Compare commits
129
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
06d4f45e18 | ||
|
|
77dacfbb17 | ||
|
|
d8592d9a4f | ||
|
|
ea4a163187 | ||
|
|
fc92202e04 | ||
|
|
d8da56c6da | ||
|
|
c2b1ba4c5d | ||
|
|
04bb2d1398 | ||
|
|
ad60ba3a46 | ||
|
|
ea3eac51bd | ||
|
|
cacf537004 | ||
|
|
f4a86f5884 | ||
|
|
9f1b227fb8 | ||
|
|
6c79e456a1 | ||
|
|
0b980203fe | ||
|
|
73bf290f58 | ||
|
|
475bb4bb0b | ||
|
|
efadc691e7 | ||
|
|
4af36c69a7 | ||
|
|
bd107d2015 | ||
|
|
f007ca4227 | ||
|
|
50c4674dfa | ||
|
|
c12b870104 | ||
|
|
92aedb9365 | ||
|
|
368204115c | ||
|
|
971cdb44e2 | ||
|
|
16e4e61a0f | ||
|
|
4b36cc29a6 | ||
|
|
31ddd09087 | ||
|
|
68173de8bd | ||
|
|
960e3309bc | ||
|
|
2ee7589474 | ||
|
|
297992a4ae | ||
|
|
3f85230417 | ||
|
|
43c3158bc6 | ||
|
|
d95e9f3229 | ||
|
|
d581f9659a | ||
|
|
ea94f9234c | ||
|
|
c19468b04e | ||
|
|
c47afd8f4b | ||
|
|
afd5a3ba07 | ||
|
|
7dbad4c6d2 | ||
|
|
083a5042f6 | ||
|
|
351ca7c290 | ||
|
|
f907be2c14 | ||
|
|
6201f58c20 | ||
|
|
8fee764c73 | ||
|
|
3dd94f714d | ||
|
|
c716670be0 | ||
|
|
352066dde6 | ||
|
|
d6bd31a166 | ||
|
|
c0a4b139a2 | ||
|
|
6d60953f98 | ||
|
|
17ed06ee4d | ||
|
|
9427dac823 | ||
|
|
d74e082155 | ||
|
|
2e0013ec25 | ||
|
|
2493aa6529 | ||
|
|
85c89b16ef | ||
|
|
4f75925410 | ||
|
|
820c458f4c | ||
|
|
dc9d65524d | ||
|
|
1a4c788ed7 | ||
|
|
df8b9ac3b1 | ||
|
|
fcb1e40582 | ||
|
|
4973372612 | ||
|
|
3f6c085e8a | ||
|
|
040e3097b5 | ||
|
|
feef96ed16 | ||
|
|
4f722899df | ||
|
|
166a918b43 | ||
|
|
1b4c860f63 | ||
|
|
3b6945b2db | ||
|
|
4174e23c04 | ||
|
|
3993175e45 | ||
|
|
60c7ffc981 | ||
|
|
33de9dde4f | ||
|
|
6b85220fa4 | ||
|
|
1cbae7d41c | ||
|
|
b3a4b09f5b | ||
|
|
c8b26735c8 | ||
|
|
907e092abc | ||
|
|
3073495a06 | ||
|
|
16c411e436 | ||
|
|
2d841870a8 | ||
|
|
7f8ce7f5c0 | ||
|
|
07b5f13474 | ||
|
|
f4aac599ca | ||
|
|
2d63f0ba62 | ||
|
|
1d41682715 | ||
|
|
705765d113 | ||
|
|
16a97ed65d | ||
|
|
ecf532af0e | ||
|
|
ce600df98a | ||
|
|
b4a6a34705 | ||
|
|
a7e51b416b | ||
|
|
71f786f146 | ||
|
|
848c6fdc56 | ||
|
|
ead9ee2d13 | ||
|
|
4861d1f78d | ||
|
|
5c76c9d3ac | ||
|
|
699cbfb7c1 | ||
|
|
319d56be7e | ||
|
|
11d3678984 | ||
|
|
f55f150896 | ||
|
|
e1f1b5d18b | ||
|
|
f0fe454145 | ||
|
|
4026f6c188 | ||
|
|
29cfb8c177 | ||
|
|
a092ee9ff9 | ||
|
|
6c3cbee35c | ||
|
|
4b23db28b3 | ||
|
|
646f83642a | ||
|
|
d9bfba1e62 | ||
|
|
f1bb7747ae | ||
|
|
d2eb3e5ecf | ||
|
|
b0b1cf4f01 | ||
|
|
d91ea41f64 | ||
|
|
5fed429827 | ||
|
|
9a569bfb8d | ||
|
|
e2c41ffa5a | ||
|
|
13e4ed8515 | ||
|
|
c66fe11e16 | ||
|
|
75d0eb32be | ||
|
|
c44b9f91cd | ||
|
|
82da252cf0 | ||
|
|
ce72b67cef | ||
|
|
2a94d16834 | ||
|
|
da7baa5e39 |
@@ -1 +1,4 @@
|
||||
vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -172,6 +173,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
|
||||
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
for _, missing := range readBuckets {
|
||||
if missing == delivery.PlacementBucket {
|
||||
continue // it starts without it, below
|
||||
}
|
||||
b := startBed(t, missing)
|
||||
select {
|
||||
case err := <-b.start(t):
|
||||
@@ -218,6 +222,34 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
t.Fatal("the put was not taken further once its bucket was made")
|
||||
}
|
||||
}
|
||||
|
||||
// Review of L2 (D1): without the indexer's placement state it starts all the same, and once the bucket is made (the
|
||||
// indexer and the assigner come in one delivery) it is read with no restart: a module only the placement names is
|
||||
// said as placed by one source only.
|
||||
pb := startBed(t, delivery.PlacementBucket)
|
||||
pdone := pb.start(t)
|
||||
select {
|
||||
case err := <-pdone:
|
||||
t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
plkey, _ := delivery.PlacementKey("n1")
|
||||
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the placement state made after the start was not read")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// putJSON writes a value into one of the bed's buckets.
|
||||
@@ -265,6 +297,10 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
|
||||
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
|
||||
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
|
||||
func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
// The placement answers' bound (ADR 0559 §5), shortened before the program starts and put back after it stops.
|
||||
bound := assign.AnswerBound
|
||||
t.Cleanup(func() { assign.AnswerBound = bound })
|
||||
assign.AnswerBound = 1500 * time.Millisecond
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
@@ -316,6 +352,63 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
|
||||
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
|
||||
}
|
||||
|
||||
// ADR 0559 §5 (issue 1126): a catalogue pull request that places g14 for the first time (the seed's case) is composed
|
||||
// with its candidate placement, touches g14, and waits for g14's answer at its head; g14 answering the same passes it.
|
||||
// A node that does not answer within the bound fails the check, named, at the bound's end with no other event; its later
|
||||
// answer at the same head passes the same heads' composition.
|
||||
record, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"}})
|
||||
b.putJSON(t, delivery.HaveBucket, "g14.node", record)
|
||||
placed := func(group string, number int, head string) (string, uint64) {
|
||||
t.Helper()
|
||||
value, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head},
|
||||
Placement: &delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementCandidateKey(group, "g14")
|
||||
rev, err := b.buckets[delivery.PlacementCandidatesBucket].Put(b.ctx, key, value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pull := []delivery.CandidatePull{{Head: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head}, Modules: []string{}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pull[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: hash, Pulls: pull, FPSet: delivery.FingerprintSet(pull)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey(group)
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
key, _ = delivery.CompositionKey(group, hash)
|
||||
return key, rev
|
||||
}
|
||||
answer := func(group, head string, rev uint64) {
|
||||
t.Helper()
|
||||
have, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"},
|
||||
Candidates: map[string]delivery.CandidateAnswer{group: {Head: head, Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"web"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, "g14.node", have)
|
||||
}
|
||||
seedHead, lateHead := strings.Repeat("5", 40), strings.Repeat("6", 40)
|
||||
seed, seedRev := placed("feat/seed", 20, seedHead)
|
||||
waiting := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waiting.Words, "g14") {
|
||||
t.Errorf("the seed's composition waits saying %q, want g14 named", waiting.Words)
|
||||
}
|
||||
answer("feat/seed", seedHead, seedRev)
|
||||
if v := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass }); !slices.Contains(v.Machines, "g14") {
|
||||
t.Errorf("the seed's composition = %+v, want g14 composed", v)
|
||||
}
|
||||
late, lateRev := placed("feat/late", 21, lateHead)
|
||||
failed := b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionFail })
|
||||
if failed.Machine != "g14" || !strings.Contains(failed.Words, delivery.CompositionUnanswered) {
|
||||
t.Errorf("a node silent past its bound: %+v", failed)
|
||||
}
|
||||
answer("feat/late", lateHead, lateRev)
|
||||
b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass })
|
||||
}
|
||||
|
||||
// get waits for a key of one of the bed's buckets and answers its value.
|
||||
@@ -406,6 +499,32 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if !cleared {
|
||||
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the
|
||||
// assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and
|
||||
// its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn).
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PlacementKey(node)
|
||||
b.putJSON(t, delivery.PlacementBucket, pkey, pl)
|
||||
}
|
||||
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
|
||||
t.Errorf("ace's want once placed = %+v", w)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
var self bool
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
nw, err := delivery.DecodeNodeWant(e.Value())
|
||||
self = err == nil && nw.Self
|
||||
}
|
||||
}
|
||||
if !self {
|
||||
t.Error("ace's own want does not say it composes its own once placed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -59,6 +60,8 @@ var readBuckets = []string{
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
@@ -98,6 +101,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PlacementBucket && err != nil {
|
||||
// The indexer's placement state comes with the second milestone (ADR 0549 §2): without it the controller's
|
||||
// assignments alone name where modules run, as before any placement was read (ADR 0549 §5).
|
||||
logf("the bucket %s cannot be opened (%v): the controller's assignments alone are read until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
@@ -151,7 +160,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
}
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
@@ -169,6 +178,8 @@ var watched = []string{
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
@@ -179,7 +190,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -223,7 +234,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
w.put(name, e.Key(), value)
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
wake()
|
||||
}
|
||||
}
|
||||
@@ -263,7 +274,12 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && (name == delivery.PlacementBucket || name == delivery.PlacementCandidatesBucket) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
@@ -275,7 +291,29 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
for range len(watched) + 1 {
|
||||
// The indexer's placement state arrives with the same delivery as this assigner (ADR 0549 §2): when it is not on the
|
||||
// bus at this start it is bound once the bus says it was created, and watched from then on, with no restart (issue
|
||||
// 1790's kind). Until then the controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, 2)
|
||||
if kvs[delivery.PlacementBucket] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, delivery.PlacementBucket, "the controller's assignments alone are read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementBucket, kv} })
|
||||
}
|
||||
// So are the placement candidates (ADR 0559 §3), with the same delivery as the indexer that writes them.
|
||||
if kvs[delivery.PlacementCandidatesBucket] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, delivery.PlacementCandidatesBucket, "no change's placement is composed",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementCandidatesBucket, kv} })
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
kvs[a.name] = a.kv
|
||||
logger.Printf("the bucket %s is open and watched", a.name)
|
||||
return watch(a.name)
|
||||
}
|
||||
for range started + 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -285,6 +323,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.wakeAt = wakeAtVia(ctx, wake)
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
@@ -297,6 +336,10 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
case err := <-errs:
|
||||
return err
|
||||
case <-settled:
|
||||
case kv := <-placementAppeared:
|
||||
if err := takePlacement(kv); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
p.pass(ctx)
|
||||
@@ -304,6 +347,31 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
}
|
||||
|
||||
// wakeAtVia asks wake once at each time it is given, as a context's deadline (ADR 0437 §8: no timer), each time once.
|
||||
func wakeAtVia(ctx context.Context, wake func()) func(time.Time) {
|
||||
var mu sync.Mutex
|
||||
asked := map[int64]bool{}
|
||||
return func(at time.Time) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if asked[at.UnixNano()] {
|
||||
return
|
||||
}
|
||||
asked[at.UnixNano()] = true
|
||||
go func() {
|
||||
bound, cancel := context.WithDeadline(ctx, at)
|
||||
defer cancel()
|
||||
<-bound.Done()
|
||||
mu.Lock()
|
||||
delete(asked, at.UnixNano())
|
||||
mu.Unlock()
|
||||
if ctx.Err() == nil {
|
||||
wake()
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||
var PutsReopenEvery = time.Minute
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
@@ -170,3 +170,5 @@ type kvEntry struct {
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
func (e kvEntry) Revision() uint64 { return 1 }
|
||||
func (e kvEntry) Created() time.Time { return time.Time{} }
|
||||
|
||||
@@ -3,7 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -40,6 +40,12 @@ type passer struct {
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// wakeAt asks a pass at a time: the end of a placement candidate's answer bound (ADR 0559 §5), as a context's deadline
|
||||
// (ADR 0437 §8: no timer). Nil asks none.
|
||||
wakeAt func(at time.Time)
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
@@ -61,7 +67,23 @@ func (p *passer) pass(ctx context.Context) {
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
placed := p.w.placedNodes()
|
||||
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
|
||||
for node := range placed {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
if placed[node] {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -98,6 +120,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
names := map[string]bool{}
|
||||
@@ -114,12 +138,20 @@ func (p *passer) wants(ctx context.Context) {
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if placed[node] {
|
||||
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
|
||||
// names the build alone, and the node judges the move itself.
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
|
||||
}
|
||||
return p.w.composed(node, name, fp)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
@@ -153,8 +185,11 @@ func (p *passer) wants(ctx context.Context) {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
p.yeses(ctx, name, m, placed, wanted[name], &out)
|
||||
gatherNeither(neither, name, out)
|
||||
leftNeither(left, m, wanted[name], out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
@@ -182,6 +217,9 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
if frozen[node+"."+name] {
|
||||
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
|
||||
}
|
||||
key, err := delivery.WantKey(node, name)
|
||||
if err != nil {
|
||||
p.logf("no want of %s on %s is written: %v", name, node, err)
|
||||
@@ -210,7 +248,7 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
reason = h.Reason()
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
@@ -229,6 +267,60 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
}
|
||||
|
||||
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
|
||||
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
|
||||
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
|
||||
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
|
||||
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
|
||||
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
|
||||
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
|
||||
if p.nodeFailed == nil {
|
||||
p.nodeFailed = map[string]string{}
|
||||
}
|
||||
look := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
look[node] = true
|
||||
}
|
||||
for key := range p.nodeFailed {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
if first {
|
||||
for node := range known.Online {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(look)) {
|
||||
key := node + "." + module
|
||||
f, failing := out.NodeFailed[node]
|
||||
last, said := p.nodeFailed[key]
|
||||
switch {
|
||||
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
|
||||
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
|
||||
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
|
||||
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
|
||||
case !failing && (said || first):
|
||||
if said {
|
||||
p.logf("%s on %s no longer fails its health check alone", module, node)
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
|
||||
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
delete(p.nodeFailed, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
@@ -242,6 +334,48 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||
// left it with none).
|
||||
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; on {
|
||||
continue
|
||||
}
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
why := ""
|
||||
h, ruled := out.Rules[node]
|
||||
switch {
|
||||
case written && !w.Removed:
|
||||
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||
why = "it is held, and its held condition says so"
|
||||
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||
why = "its move waits for the operator's yes on its record ask"
|
||||
case putOn(m, node):
|
||||
why = "its put waits on its own steps and the operator's answer"
|
||||
case m.FP == "":
|
||||
why = "the trunk has no build of it"
|
||||
case ruled:
|
||||
why = h.Reason()
|
||||
default:
|
||||
why = "the node does not run it"
|
||||
}
|
||||
into[node+"."+m.Name] = why
|
||||
}
|
||||
}
|
||||
|
||||
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||
func putOn(m assign.Module, node string) bool {
|
||||
p, put := m.Puts[node]
|
||||
_, owns := assign.PutOwns(p)
|
||||
return put && owns
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
@@ -258,7 +392,9 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||
// it does not hold is no longer assigned.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
@@ -278,21 +414,29 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Reason())
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
why, assigned := left[key]
|
||||
if !assigned {
|
||||
why = "it is no longer assigned there"
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
if why == "" {
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
} else {
|
||||
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||
}
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
@@ -337,6 +481,9 @@ func (p *passer) compositions(ctx context.Context) {
|
||||
} else {
|
||||
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
|
||||
}
|
||||
if ch.Placement != nil && v.State != delivery.CompositionWaits {
|
||||
v = p.answers(*mesh, ch, v)
|
||||
}
|
||||
}
|
||||
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
|
||||
before.Took, before.V = v.Took, 0
|
||||
@@ -348,6 +495,40 @@ func (p *passer) compositions(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// answers judges every placed node's answer to the change's placement into v (ADR 0559 §5). While a node has not answered
|
||||
// within the bound the verdict waits, and the pass at the bound's end is asked (wakeAt), so a silent node is failed then,
|
||||
// named, and a later answer of the same head is judged on its own have's change.
|
||||
func (p *passer) answers(mesh assign.Mesh, ch assign.Change, v delivery.CompositionVerdict) delivery.CompositionVerdict {
|
||||
trunkPlaced := map[string]delivery.Placement{}
|
||||
for key, value := range p.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
|
||||
trunkPlaced[node] = pl
|
||||
}
|
||||
}
|
||||
fps := map[string]string{}
|
||||
for name, m := range p.w.trunk() {
|
||||
if m.FP2 != "" {
|
||||
fps[name] = m.FP2
|
||||
} else if m.FP != "" {
|
||||
fps[name] = m.FP
|
||||
}
|
||||
}
|
||||
intended, notes := assign.Intended(mesh.Input, trunkPlaced, ch.Placement, fps)
|
||||
a := assign.JudgeAnswers(ch.Placement, assign.PlacedNodes(trunkPlaced, ch.Placement), p.w.candidateAnswers(ch.Group), intended, p.w.online(), p.clock())
|
||||
v = assign.WithAnswers(v, a)
|
||||
if v.State != delivery.CompositionWaits {
|
||||
v.Warnings = append(v.Warnings, notes...)
|
||||
}
|
||||
if a.Waits && p.wakeAt != nil {
|
||||
p.wakeAt(a.Until)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
|
||||
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
|
||||
value, err := encode()
|
||||
@@ -675,3 +856,83 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
|
||||
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
|
||||
// for them and keeps the one written before. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
|
||||
frozen := map[string]bool{}
|
||||
now := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
frozen[d.Node+"."+d.Module] = true
|
||||
c := assign.PlacementCondition(d, true)
|
||||
now[c.Key] = true
|
||||
if p.said[c.Key+".raised"] {
|
||||
continue
|
||||
}
|
||||
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
|
||||
continue
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
}
|
||||
prefix := delivery.AssignerModule + ".placement."
|
||||
for key := range p.said {
|
||||
c, raised := strings.CutSuffix(key, ".raised")
|
||||
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
|
||||
continue
|
||||
}
|
||||
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
|
||||
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
delete(p.said, key)
|
||||
}
|
||||
return frozen
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, placed map[string]bool, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if !placed[node] {
|
||||
continue
|
||||
}
|
||||
was, had := before[node]
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
if !had || was.Removed {
|
||||
continue
|
||||
}
|
||||
w = was
|
||||
}
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes := ""
|
||||
if had && was.FP == w.FP {
|
||||
yes = was.Yes
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
if m.Allowed != nil && m.Allowed(node, h.Waits) {
|
||||
yes = h.Waits
|
||||
} else {
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes != w.Yes || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Composed, w.Self = yes, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,6 +342,9 @@ func putOfTheBus(t *testing.T) {
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
@@ -357,30 +360,66 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil, nil) // the say fails
|
||||
p.sayNeither(one, nil, nil)
|
||||
p.sayNeither(one, nil, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
wants := map[string]string{"g14.screen-lock": ""}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||
t.Errorf("the clear without a want said %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
puts map[string]delivery.Put
|
||||
before map[string]delivery.Want
|
||||
why string
|
||||
}{
|
||||
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||
{"nothing", nil, nil, "the node does not run it"},
|
||||
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||
} {
|
||||
left := map[string]string{}
|
||||
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||
}
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
@@ -399,6 +438,7 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
@@ -406,12 +446,71 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
|
||||
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
|
||||
for i, c := range []struct {
|
||||
failed map[string]assign.NodeFailure
|
||||
state string // the condition said in this pass, "" for none
|
||||
}{
|
||||
{failing, asks.StateRaised},
|
||||
{failing, ""},
|
||||
{nil, asks.StateCleared},
|
||||
{nil, ""},
|
||||
} {
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
|
||||
switch {
|
||||
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
|
||||
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
|
||||
case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
|
||||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
|
||||
t.Errorf("node failed, raised %+v", conds[0])
|
||||
}
|
||||
}
|
||||
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
|
||||
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
|
||||
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
|
||||
}
|
||||
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
|
||||
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
known assign.Nodes
|
||||
keys []string
|
||||
}{
|
||||
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
} {
|
||||
conds = nil
|
||||
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
|
||||
fresh.logf = func(string, ...any) {}
|
||||
fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
|
||||
var keys []string
|
||||
for _, cond := range conds {
|
||||
if cond.State == asks.StateCleared {
|
||||
keys = append(keys, cond.Key)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
|
||||
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
@@ -449,7 +548,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
@@ -665,6 +764,8 @@ func mustKey(key string, err error) string {
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
@@ -713,3 +814,133 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
|
||||
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
|
||||
// assignments are the one source and nothing is raised.
|
||||
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
assigned := func(node, module string) {
|
||||
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
|
||||
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
|
||||
}
|
||||
place := func(node string, modules ...string) {
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, err := delivery.EncodePlacement(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
assigned("g14", "a")
|
||||
assigned("g14", "b")
|
||||
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v, said %v", got, said)
|
||||
}
|
||||
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
|
||||
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
place("g14", "a", "c")
|
||||
got, _ := p.w.assigned()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
p.sayPlacement(p.w.placementDiffers())
|
||||
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
|
||||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
|
||||
!strings.Contains(said[1].Explanation, "the catalogue's placement names c on g14") {
|
||||
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
|
||||
}
|
||||
place("g14", "a", "b", "c")
|
||||
assigned("g14", "c")
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
|
||||
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
|
||||
// wrote its record; until then it says why and composes the assignments alone.
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
|
||||
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
|
||||
}
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
|
||||
p.w.put(delivery.HaveBucket, "g14.node", rec)
|
||||
m := p.w.mesh(context.Background(), none)
|
||||
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
|
||||
// kept there while the want stays at its build.
|
||||
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
sync := func() {
|
||||
for k, v := range wantsKV.values {
|
||||
put(delivery.AssignerWantsBucket, k, v)
|
||||
}
|
||||
}
|
||||
|
||||
p.wants(context.Background())
|
||||
p.nodeWants(context.Background())
|
||||
sync()
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
|
||||
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("g14")
|
||||
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
|
||||
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
|
||||
}
|
||||
|
||||
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
|
||||
put(delivery.HaveBucket, "g14.web", waits)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
sync()
|
||||
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
|
||||
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
|
||||
}
|
||||
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
|
||||
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", ran)
|
||||
p.wants(context.Background())
|
||||
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
|
||||
t.Errorf("a pass after the move dropped the yes: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
@@ -46,6 +48,14 @@ type world struct {
|
||||
healthy map[string][]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
|
||||
meta map[string]entryMeta
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
type entryMeta struct {
|
||||
revision uint64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -56,7 +66,27 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
|
||||
w.put(bucket, key, value)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if value == nil {
|
||||
delete(w.meta, bucket+"|"+key)
|
||||
return
|
||||
}
|
||||
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
|
||||
}
|
||||
|
||||
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
|
||||
func (w *world) metaOf(bucket, key string) entryMeta {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.meta[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -162,8 +192,56 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||
// assigned is each node's modules both the controller's assignments and the indexer's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
byAssignment, all := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the indexer's placement do not both name.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
byAssignment, _ := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded)
|
||||
return differ
|
||||
}
|
||||
|
||||
// placed is each node's modules in the indexer's placement (ADR 0549 §2), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
values := w.read(delivery.PlacementBucket)
|
||||
for key, value := range values {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil || p.Node != node {
|
||||
continue
|
||||
}
|
||||
out[node] = p.Assigned()
|
||||
}
|
||||
return out, len(values) > 0
|
||||
}
|
||||
|
||||
// placedNodes is every node the indexer's placement names (ADR 0549 §2): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string]bool, len(placed))
|
||||
for node := range placed {
|
||||
out[node] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
|
||||
// name order.
|
||||
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
@@ -210,9 +288,86 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
|
||||
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
|
||||
// assignments alone are composed, as before any placement.
|
||||
if _, seeded := w.placed(); seeded {
|
||||
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
|
||||
if err != nil {
|
||||
m.InputUnread = err.Error()
|
||||
} else {
|
||||
m.Input = &in
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// worldReader is what the assigner watched as frombus reads the bus: the indexer's placement and modules, each node's
|
||||
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
|
||||
// composition it asks resolves, and stands in for what it does not carry).
|
||||
type worldReader struct {
|
||||
w *world
|
||||
manifests getter
|
||||
}
|
||||
|
||||
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
out := map[string]delivery.Placement{}
|
||||
for key, value := range r.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePlacement(value); err == nil && p.Node == node {
|
||||
out[node] = p
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range r.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
return r.w.trunk(), nil
|
||||
}
|
||||
|
||||
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := r.manifests(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
func (worldReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
|
||||
|
||||
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
@@ -262,9 +417,92 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
|
||||
sort.Strings(ch.Twice[name])
|
||||
}
|
||||
}
|
||||
cp, waits := w.placementOf(ctx, manifests, c)
|
||||
if waits != "" {
|
||||
return ch, waits
|
||||
}
|
||||
ch.Placement = cp
|
||||
return ch, ""
|
||||
}
|
||||
|
||||
// placementOf is the change's own placement (ADR 0559 §5): the indexer's placement candidates of the group's catalogue
|
||||
// pull request, with the input the nodes compose from with them laid over the trunk's placement. None when the group has
|
||||
// no catalogue pull request or it changes no placement file. It waits while a key names another head than the group's.
|
||||
func (w *world) placementOf(ctx context.Context, manifests getter, c delivery.Candidate) (*assign.ChangePlacement, string) {
|
||||
var pull *delivery.CandidatePull
|
||||
for i := range c.Pulls {
|
||||
if delivery.SameRepository(c.Pulls[i].Repository, delivery.CatalogueRepository) {
|
||||
pull = &c.Pulls[i]
|
||||
}
|
||||
}
|
||||
if pull == nil {
|
||||
return nil, ""
|
||||
}
|
||||
cp := &assign.ChangePlacement{Placements: map[string]*delivery.Placement{}, Revisions: map[string]uint64{}, Head: pull.Head.Head}
|
||||
for key, value := range w.read(delivery.PlacementCandidatesBucket) {
|
||||
group, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || group != c.Group {
|
||||
continue
|
||||
}
|
||||
pc, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil || pc.Pull.Number != pull.Number || !delivery.SameRepository(pc.Pull.Repository, pull.Repository) {
|
||||
continue
|
||||
}
|
||||
if pc.Pull.Head != pull.Head.Head {
|
||||
return nil, fmt.Sprintf("the indexer holds the placement candidates of %s#%d at %s, not yet at its head %s", pull.Repository, pull.Number, pc.Pull.Head, pull.Head.Head)
|
||||
}
|
||||
meta := w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
cp.Placements[node], cp.Revisions[node] = pc.Placement, meta.revision
|
||||
if meta.at.After(cp.Written) {
|
||||
cp.Written = meta.at
|
||||
}
|
||||
}
|
||||
if len(cp.Placements) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true, Placements: cp.Placements})
|
||||
if err != nil {
|
||||
cp.Unread = err.Error()
|
||||
} else {
|
||||
cp.Input = &in
|
||||
}
|
||||
return cp, ""
|
||||
}
|
||||
|
||||
// candidateAnswers is, by node, each node's answer to the group's placement candidates in its `<node>.node` have.
|
||||
func (w *world) candidateAnswers(group string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[group]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
out[node] = p.Online
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// wantsOf is the wants the assigner wrote, by module then node.
|
||||
func (w *world) wantsOf() map[string]map[string]delivery.Want {
|
||||
out := map[string]map[string]delivery.Want{}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f h1:aWqn4HHTPUp/+28h7b8uFqQptAYvQFs05CALMHN7SkI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -81,6 +81,29 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
@@ -96,6 +119,14 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
if h.Bus {
|
||||
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||
asks.NeedsYou, needs, module, node, h.Words)}
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
@@ -103,6 +134,17 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A change's own placement (novox/hq ADR 0559 §5, issue 1126): a catalogue pull request that changes placement files has
|
||||
// the indexer's placement candidates, one key `<group>.<node>` for each node whose file at its head differs from the
|
||||
// trunk's. The check composes the change with those placements laid over the trunk's (frombus, Scope.Placements), counts
|
||||
// every such node among the machines it touches, and then reads each placed node's own answer in its `<node>.node`
|
||||
// have: it fails on what a node composes differently that the change does not intend, on a refusal, and on a node
|
||||
// placed at the head or on the trunk that has not answered at the head within AnswerBound of the keys being written.
|
||||
|
||||
// AnswerBound is how long the check waits for every placed node's answer after the candidate keys were written (ADR 0559
|
||||
// §5): a node asleep or away is named past it, never passed. A variable so a test on the bus can shorten it.
|
||||
var AnswerBound = 10 * time.Minute
|
||||
|
||||
// ChangePlacement is the change's own placement, read from the indexer's placement candidates of its group.
|
||||
type ChangePlacement struct {
|
||||
// Placements is by node the candidate placement; nil when the head removes the node's file.
|
||||
Placements map[string]*delivery.Placement
|
||||
// Revisions is by node the revision of its candidate key: the answer to it names that revision.
|
||||
Revisions map[string]uint64
|
||||
// Head is the catalogue pull request's head the candidates were read at, and Written when the newest key was written.
|
||||
Head string
|
||||
Written time.Time
|
||||
// Input is the trunk's composition input with Placements laid over it (frombus.Input, Scope.Placements), its catalogue
|
||||
// the trunk's; nil with Unread saying why when it could not be read.
|
||||
Input *compose.Input
|
||||
Unread string
|
||||
}
|
||||
|
||||
// Nodes is every node the change's placement names, sorted.
|
||||
func (p *ChangePlacement) Nodes() []string {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(p.Placements))
|
||||
for node := range p.Placements {
|
||||
out = append(out, node)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Intended is, by node the trunk places, the modules whose composition the change's placement changes: those it adds,
|
||||
// removes or whose placed entry differs, and those whose composition (with the trunk's catalogue, the same stand-ins
|
||||
// either side) differs between the trunk's input and the change's (delivery.CompareComposed), the node's own named
|
||||
// "node". A node the trunk does not place yet has none (ADR 0559 §5: at the seed every node must answer the same). fps
|
||||
// gives the build each module is composed at, the trunk's. A node whose compositions cannot be made either side keeps
|
||||
// the modules its placement entries name, and is said in notes.
|
||||
func Intended(trunk *compose.Input, trunkPlaced map[string]delivery.Placement, cp *ChangePlacement, fps map[string]string) (intended map[string]map[string]bool, notes []string) {
|
||||
intended = map[string]map[string]bool{}
|
||||
if cp == nil {
|
||||
return intended, nil
|
||||
}
|
||||
for _, node := range cp.Nodes() {
|
||||
before, placed := trunkPlaced[node]
|
||||
if !placed {
|
||||
continue
|
||||
}
|
||||
set := map[string]bool{}
|
||||
after := cp.Placements[node]
|
||||
var afterModules []delivery.PlacedModule
|
||||
if after != nil {
|
||||
afterModules = after.Modules
|
||||
}
|
||||
entries := map[string]delivery.PlacedModule{}
|
||||
for _, m := range before.Modules {
|
||||
entries[m.Module] = m
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range afterModules {
|
||||
seen[m.Module] = true
|
||||
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
|
||||
set[m.Module] = true
|
||||
}
|
||||
}
|
||||
for name := range entries {
|
||||
if !seen[name] {
|
||||
set[name] = true
|
||||
}
|
||||
}
|
||||
if trunk != nil && cp.Input != nil {
|
||||
was, _, _, errB := compose.CompositionsShadow(*trunk, node, fps)
|
||||
now, _, _, errA := compose.CompositionsShadow(*cp.Input, node, fps)
|
||||
if errA == nil && errB == nil {
|
||||
running := make([]delivery.Composed, 0, len(was))
|
||||
for _, c := range was {
|
||||
running = append(running, c)
|
||||
}
|
||||
cmp := delivery.CompareComposed(now, running)
|
||||
for _, d := range cmp.Differs {
|
||||
set[d.Module] = true
|
||||
}
|
||||
for _, d := range cmp.Added {
|
||||
set[d.Module] = true
|
||||
}
|
||||
for _, m := range cmp.Removed {
|
||||
set[m] = true
|
||||
}
|
||||
} else {
|
||||
notes = append(notes, fmt.Sprintf("what the change intends on %s is read from its placement file alone: its compositions cannot be made (trunk: %v; change: %v)", node, errB, errA))
|
||||
}
|
||||
}
|
||||
intended[node] = set
|
||||
}
|
||||
return intended, notes
|
||||
}
|
||||
|
||||
// Answered is what the placed nodes answered: the failures to name, the nodes that have not answered, and the notes.
|
||||
type Answered struct {
|
||||
Fails []string
|
||||
Silent []string
|
||||
Machine string
|
||||
Waits bool
|
||||
Until time.Time
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// JudgeAnswers reads each placed node's answer to the change's placement (ADR 0559 §5). placed is every node placed at
|
||||
// the head or on the trunk; answers is by node its HaveNode.Candidates entry for the group, nil when it has none; online
|
||||
// says whether the watchman holds the node online. An answer counts when it names the candidates' head and, for a node
|
||||
// with a key of its own, that key's revision. It fails on a refusal, on a module that differs, is added or is removed
|
||||
// that intended does not name, and, from Written plus AnswerBound on, on every node with no answer that counts, named,
|
||||
// asleep or not; before that the check waits for them.
|
||||
func JudgeAnswers(cp *ChangePlacement, placed []string, answers map[string]*delivery.CandidateAnswer, intended map[string]map[string]bool, online map[string]bool, now time.Time) Answered {
|
||||
var out Answered
|
||||
if cp == nil {
|
||||
return out
|
||||
}
|
||||
failAt := func(node, words string) {
|
||||
if out.Machine == "" {
|
||||
out.Machine = node
|
||||
}
|
||||
out.Fails = append(out.Fails, words)
|
||||
}
|
||||
for _, node := range placed {
|
||||
a := answers[node]
|
||||
rev, keyed := cp.Revisions[node]
|
||||
if a == nil || a.Head != cp.Head || (keyed && a.Revision != rev) {
|
||||
out.Silent = append(out.Silent, node)
|
||||
continue
|
||||
}
|
||||
if a.Refused != "" {
|
||||
failAt(node, fmt.Sprintf("%s cannot compose the candidate placement: %s", node, a.Refused))
|
||||
continue
|
||||
}
|
||||
for _, r := range a.Refusals {
|
||||
failAt(node, fmt.Sprintf("%s refuses the candidate placement for %s: %s", node, r.Module, r.Why))
|
||||
}
|
||||
want := intended[node]
|
||||
for _, d := range a.Differs {
|
||||
if !want[d.Module] {
|
||||
failAt(node, fmt.Sprintf("%s composes %s differently with the candidate placement, which the change does not intend: %s", node, d.Module, d.Why))
|
||||
}
|
||||
}
|
||||
for _, d := range a.Added {
|
||||
if !want[d.Module] {
|
||||
failAt(node, fmt.Sprintf("%s would run %s with the candidate placement, which the change does not intend", node, d.Module))
|
||||
}
|
||||
}
|
||||
for _, m := range a.Removed {
|
||||
if !want[m] {
|
||||
failAt(node, fmt.Sprintf("%s would stop running %s with the candidate placement, which the change does not intend", node, m))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out.Silent) == 0 {
|
||||
return out
|
||||
}
|
||||
out.Until = cp.Written.Add(AnswerBound)
|
||||
named := make([]string, len(out.Silent))
|
||||
for i, node := range out.Silent {
|
||||
named[i] = node
|
||||
if !online[node] {
|
||||
named[i] = node + " (offline)"
|
||||
}
|
||||
}
|
||||
if now.Before(out.Until) {
|
||||
out.Waits = true
|
||||
out.Warnings = append(out.Warnings, fmt.Sprintf("waits for the answers of %s to the candidate placement at %s, until %s",
|
||||
strings.Join(named, ", "), short(cp.Head), out.Until.UTC().Format(time.RFC3339)))
|
||||
return out
|
||||
}
|
||||
for _, node := range out.Silent {
|
||||
if out.Machine == "" {
|
||||
out.Machine = node
|
||||
}
|
||||
}
|
||||
out.Fails = append(out.Fails, fmt.Sprintf("%s %s (%s after the candidate keys were written at %s)",
|
||||
strings.Join(named, ", "), delivery.CompositionUnanswered, AnswerBound, cp.Written.UTC().Format(time.RFC3339)))
|
||||
return out
|
||||
}
|
||||
|
||||
// PlacedNodes is every node placed at the head or on the trunk: the trunk's placed nodes with the change's placement laid
|
||||
// over them (a node whose file the head removes stays, as the trunk places it), sorted.
|
||||
func PlacedNodes(trunkPlaced map[string]delivery.Placement, cp *ChangePlacement) []string {
|
||||
set := map[string]bool{}
|
||||
for node := range trunkPlaced {
|
||||
set[node] = true
|
||||
}
|
||||
if cp != nil {
|
||||
for node, p := range cp.Placements {
|
||||
if p != nil {
|
||||
set[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for node := range set {
|
||||
out = append(out, node)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// WithAnswers is v with the placed nodes' answers judged in: a fail adds its failures (the words of nodes not heard end
|
||||
// with delivery.CompositionUnanswered, so the checker takes a later composition of the same heads in when they answer), a
|
||||
// wait makes a passing verdict wait (a composition that waits composed no machine), and a failing one stands.
|
||||
func WithAnswers(v delivery.CompositionVerdict, a Answered) delivery.CompositionVerdict {
|
||||
if len(a.Fails) > 0 {
|
||||
if v.State == delivery.CompositionFail {
|
||||
v.Words += "; " + strings.Join(a.Fails, "; ")
|
||||
} else {
|
||||
v.State, v.Words, v.Machine = delivery.CompositionFail, strings.Join(a.Fails, "; "), a.Machine
|
||||
}
|
||||
return v
|
||||
}
|
||||
if a.Waits && v.State == delivery.CompositionPass {
|
||||
return delivery.CompositionVerdict{Group: v.Group, Heads: v.Heads, State: delivery.CompositionWaits, SDK: v.SDK, Took: v.Took,
|
||||
Words: strings.Join(a.Warnings, "; ")}
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -35,6 +35,13 @@ type Mesh struct {
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
|
||||
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
|
||||
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
|
||||
Input *compose.Input
|
||||
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
|
||||
// nodes and their assignments alone, and says so in its warnings.
|
||||
InputUnread string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
@@ -52,12 +59,15 @@ type Change struct {
|
||||
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
|
||||
Digests map[string]string
|
||||
Members map[string]string
|
||||
// Placement is the change's own placement, read from the indexer's placement candidates of the group (ADR 0559 §5);
|
||||
// nil when the group changes no placement file.
|
||||
Placement *ChangePlacement
|
||||
}
|
||||
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine.
|
||||
// touches no machine. Every node whose placement the change changes is touched too (ADR 0559 §5, changing ADR 0446 §3).
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
@@ -91,6 +101,11 @@ func Touched(m Mesh, c Change) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, node := range c.Placement.Nodes() {
|
||||
if !slices.Contains(out, node) {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -163,6 +178,25 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit
|
||||
delete(changed, name)
|
||||
}
|
||||
after := input(changed, m.Assigned)
|
||||
if c.Placement != nil {
|
||||
// A change of placement is composed over the input the nodes compose from, its placement laid over the trunk's
|
||||
// (frombus, ADR 0559 §5): the trunk as the nodes compose it where the catalogue places them, the change's
|
||||
// manifests over the trunk's catalogue.
|
||||
if m.Input != nil {
|
||||
trunk = *m.Input
|
||||
trunk.Catalogue = withOverlay(m.Input.Catalogue, m.Manifests)
|
||||
}
|
||||
if c.Placement.Input == nil {
|
||||
failAt("", "the change's placement cannot be composed: the input with it cannot be read from the bus: "+c.Placement.Unread)
|
||||
} else {
|
||||
after = *c.Placement.Input
|
||||
cat := withOverlay(c.Placement.Input.Catalogue, changed)
|
||||
for _, name := range c.Removed {
|
||||
delete(cat, name)
|
||||
}
|
||||
after.Catalogue = cat
|
||||
}
|
||||
}
|
||||
for _, node := range machines {
|
||||
_, before := compose.Resolve(trunk, node)
|
||||
_, now := compose.Resolve(after, node)
|
||||
@@ -272,6 +306,18 @@ func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// withOverlay is base with every manifest of over laid over it, a copy.
|
||||
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
|
||||
out := make(map[string]catalogue.Manifest, len(base)+len(over))
|
||||
for name, man := range base {
|
||||
out[name] = man
|
||||
}
|
||||
for name, man := range over {
|
||||
out[name] = man
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -57,6 +58,79 @@ func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
|
||||
// ADR 0559 §5 (issue 1126): a change of placement composes its placement laid over the trunk's input and fails naming
|
||||
// the node it breaks: a module no manifest names placed on g14; ace,
|
||||
// is no longer composed with it; a placement whose input cannot be read fails; a placement that breaks nothing passes, g14 touched.
|
||||
m := meshOf()
|
||||
placed := func(assigned map[string][]string) *ChangePlacement {
|
||||
in := input(m.Manifests, assigned)
|
||||
return &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": {Node: "g14"}}, Input: &in}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
cp *ChangePlacement
|
||||
state string
|
||||
machine string
|
||||
words string
|
||||
}{
|
||||
{"a placement that breaks g14", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web", "ghost"}}), delivery.CompositionFail, "g14", "ghost"},
|
||||
{"a placement whose input is not read", &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": nil}, Unread: "no record of g14"}, delivery.CompositionFail, "", "no record of g14"},
|
||||
{"a placement that breaks nothing", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}}), delivery.CompositionPass, "", ""},
|
||||
} {
|
||||
v := Compose(m, Change{Group: "g", Heads: "0123456789ab", Placement: row.cp}, "sdk", 0)
|
||||
if v.State != row.state || v.Machine != row.machine || !strings.Contains(v.Words, row.words) || !slices.Contains(v.Machines, "g14") {
|
||||
t.Errorf("%s: %+v", row.name, v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0559 §5: the placed nodes' answers. ace is placed on the trunk and its candidate moves web there (intended);
|
||||
// g14 is not placed on the trunk (the seed), so nothing is intended there; novox is placed and unchanged.
|
||||
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
|
||||
cp := &ChangePlacement{Head: "h2", Written: at, Revisions: map[string]uint64{"ace": 7, "g14": 8},
|
||||
Placements: map[string]*delivery.Placement{"ace": {Node: "ace"}, "g14": {Node: "g14"}}}
|
||||
intended := map[string]map[string]bool{"ace": {"web": true}}
|
||||
same := func(rev uint64) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Head: "h2", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
|
||||
}
|
||||
placedNodes := []string{"ace", "g14", "novox"}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
answers map[string]*delivery.CandidateAnswer
|
||||
now time.Time
|
||||
state string
|
||||
words string
|
||||
}{
|
||||
{"all the same", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
|
||||
{"a module added where intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
|
||||
{"a module differing where not intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: "mail", Why: "the port differs"}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionFail, "ace composes mail differently"},
|
||||
{"a node's first placement file with a module differing", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: "web", Why: "a binding differs"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 composes web differently"},
|
||||
{"a refusal", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateRefused,
|
||||
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 refuses"},
|
||||
{"a node not answering, within its bound", map[string]*delivery.CandidateAnswer{"ace": same(7), "novox": same(0)}, at.Add(AnswerBound - time.Second), delivery.CompositionWaits, "g14 (offline)"},
|
||||
{"an answer of an earlier revision, past its bound", map[string]*delivery.CandidateAnswer{"ace": same(6), "g14": same(8), "novox": same(0)}, at.Add(AnswerBound), delivery.CompositionFail, "ace " + delivery.CompositionUnanswered},
|
||||
{"its later answer", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at.Add(time.Hour), delivery.CompositionPass, ""},
|
||||
} {
|
||||
pass := delivery.CompositionVerdict{Group: "g", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "sdk", Machines: []string{"ace", "g14"}}
|
||||
v := WithAnswers(pass, JudgeAnswers(cp, placedNodes, row.answers, intended, map[string]bool{"ace": true, "novox": true}, row.now))
|
||||
if v.State != row.state || !strings.Contains(v.Words, row.words) {
|
||||
t.Errorf("%s: %+v", row.name, v)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("%s: the verdict cannot be written: %v", row.name, err)
|
||||
}
|
||||
}
|
||||
trunk := map[string]delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "novox": {Node: "novox"}}
|
||||
if got := PlacedNodes(trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "g14": {Node: "g14"}}}); !slices.Equal(got, []string{"ace", "g14", "novox"}) {
|
||||
t.Errorf("placed at the head or on the trunk = %v", got)
|
||||
}
|
||||
got, _ := Intended(nil, trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}}, "g14": {Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}}}, nil)
|
||||
if !got["ace"]["mail"] || !got["ace"]["web"] || len(got["g14"]) != 0 {
|
||||
t.Errorf("intended = %v, want ace's mail and web, nothing on g14 the trunk does not place", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
@@ -109,6 +183,10 @@ func TestTouched(t *testing.T) {
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||
}
|
||||
// ADR 0559 §5: every node whose placement the change changes is touched, its file removed or not.
|
||||
if got := Touched(m, Change{Placement: &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "shanks": {Node: "shanks"}}}}); !slices.Equal(got, []string{"novox", "shanks"}) {
|
||||
t.Errorf("touched by a placement = %v", got)
|
||||
}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||
|
||||
@@ -42,9 +42,28 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: true, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
|
||||
in, fromClaims := plainInput(m.Assigned), true
|
||||
if m.Input != nil {
|
||||
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
|
||||
in = *m.Input
|
||||
in.Catalogue = nil
|
||||
fromClaims = len(in.Holdings) == 0
|
||||
}
|
||||
changeCase := composer.Case{Name: CaseChange, Manifests: change}
|
||||
if c.Placement != nil {
|
||||
// The change's own placement, laid over the input's in the change case (ADR 0559 §5; composer.Case's Placements):
|
||||
// a node the input does not hold yet is added bare, so the composer places it.
|
||||
changeCase.Placements = c.Placement.Placements
|
||||
in.Nodes = slices.Clone(in.Nodes)
|
||||
for _, node := range c.Placement.Nodes() {
|
||||
if !slices.ContainsFunc(in.Nodes, func(n compose.Node) bool { return n.Name == node }) {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: node})
|
||||
}
|
||||
}
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, changeCase}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
|
||||
@@ -195,6 +214,10 @@ func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Durati
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
if m.InputUnread != "" {
|
||||
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
|
||||
"cannot be read from the bus ("+m.InputUnread+")")
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -100,6 +101,41 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
|
||||
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
|
||||
m := digestedMesh()
|
||||
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
|
||||
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
|
||||
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
|
||||
placedJob, err := HeadJob(m, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, err := composer.Decode(placedJob.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
|
||||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
|
||||
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
|
||||
}
|
||||
|
||||
// ADR 0559 §5: a change of placement composes its own placement in the change case alone (composer.Case's Placements),
|
||||
// a node the input does not hold yet added bare so the composer places it, and the node touched.
|
||||
c.Placement = &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": {Node: "novox"}}}
|
||||
candJob, err := HeadJob(m, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cand, err := composer.Decode(candJob.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cand.Cases[0].Placements != nil || cand.Cases[1].Placements["novox"] == nil || len(cand.Input.Nodes) != 2 || cand.Input.Nodes[1].Name != "novox" ||
|
||||
!slices.Equal(cand.Machines, []string{"g14", "novox"}) || len(m.Input.Nodes) != 1 {
|
||||
t.Errorf("the placement's request = cases %+v, nodes %+v, machines %v", cand.Cases, cand.Input.Nodes, cand.Machines)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
|
||||
// the indexer's placement, read from the catalogue (ADR 0549 §5). The assigner writes wants only for a pair both name:
|
||||
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. Before the first placement
|
||||
// file reaches the indexer (no placement key at all) the controller's assignments are the one source, as before.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
type Pair struct {
|
||||
Node, Module, Only string
|
||||
}
|
||||
|
||||
// The source that names a pair alone.
|
||||
const (
|
||||
OnlyAssignments = "the controller's assignments"
|
||||
OnlyPlacement = "the catalogue's placement"
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs.
|
||||
func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) {
|
||||
if !seeded {
|
||||
return assigned, nil
|
||||
}
|
||||
set := func(m map[string][]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for node, modules := range m {
|
||||
for _, module := range modules {
|
||||
out[node+"\x00"+module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
inAssigned, inPlaced := set(assigned), set(placed)
|
||||
both := map[string][]string{}
|
||||
var differ []Pair
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
if inPlaced[node+"\x00"+module] {
|
||||
both[node] = append(both[node], module)
|
||||
} else {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
|
||||
}
|
||||
}
|
||||
}
|
||||
for node, modules := range placed {
|
||||
for _, module := range modules {
|
||||
if !inAssigned[node+"\x00"+module] {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, modules := range both {
|
||||
sort.Strings(modules)
|
||||
}
|
||||
sort.Slice(differ, func(i, j int) bool {
|
||||
if differ[i].Node != differ[j].Node {
|
||||
return differ[i].Node < differ[j].Node
|
||||
}
|
||||
return differ[i].Module < differ[j].Module
|
||||
})
|
||||
return both, differ
|
||||
}
|
||||
|
||||
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
|
||||
// both name it or neither does.
|
||||
func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
|
||||
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the catalogue's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
var needs string
|
||||
if p.Only == OnlyAssignments {
|
||||
needs = fmt.Sprintf("add %s to the catalogue's placement file of %s, or unassign it at the controller", p.Module, p.Node)
|
||||
} else {
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the merged placement file says", p.Module, p.Node)
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
|
||||
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
|
||||
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -169,6 +170,9 @@ type Outcome struct {
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
||||
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
||||
NodeFailed map[string]NodeFailure
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
@@ -192,10 +196,27 @@ type RecordNeed struct {
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why.
|
||||
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
|
||||
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
|
||||
// running an old-form build, got no want and the pass said nothing).
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
Bus bool
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
|
||||
func (h Holding) Reason() string {
|
||||
if h.Bus {
|
||||
return h.Words
|
||||
}
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
|
||||
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
|
||||
func BusBarWords(node, prev, fp string) string {
|
||||
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
@@ -211,11 +232,25 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || h.FP != m.FP || !failed(h) {
|
||||
continue
|
||||
}
|
||||
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
||||
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
||||
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
||||
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
||||
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
||||
// §2, ADR 0429 §6).
|
||||
if out.NodeFailed == nil {
|
||||
out.NodeFailed = map[string]NodeFailure{}
|
||||
}
|
||||
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
||||
continue
|
||||
}
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
}
|
||||
mm := m
|
||||
@@ -281,6 +316,13 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
ok, target = true, have.FP
|
||||
}
|
||||
}
|
||||
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
|
||||
n.Haves[node][m.Name].FP != m.FP {
|
||||
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
||||
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
||||
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
||||
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
d, _ := m.composition(node, target)
|
||||
@@ -303,6 +345,14 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
|
||||
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
|
||||
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
|
||||
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
|
||||
out.Rules[node] = h
|
||||
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
|
||||
out.Neither[node] = h
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -394,6 +444,43 @@ func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
||||
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
||||
var out []string
|
||||
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
||||
// healthy now.
|
||||
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
|
||||
out = append(out, other)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
||||
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
||||
func failedElsewhere(m Module, n Nodes, node string) bool {
|
||||
for _, other := range m.Nodes {
|
||||
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
||||
func FailedAloneWords(module, fp string, failing []string) string {
|
||||
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
||||
strings.Join(failing, ", "))
|
||||
}
|
||||
|
||||
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
||||
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
||||
type NodeFailure struct {
|
||||
Others []string
|
||||
Words string
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
@@ -466,12 +553,7 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
if _, barred := m.busBar(before, had, running); barred {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
@@ -483,6 +565,16 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
|
||||
// readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
|
||||
prev = before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
|
||||
@@ -150,6 +150,72 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
t.Errorf("g14 moved to a held fingerprint: %+v", w)
|
||||
}
|
||||
|
||||
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
|
||||
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
||||
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
|
||||
if out.Held != nil || len(out.Wants) != 0 {
|
||||
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
|
||||
}
|
||||
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
|
||||
len(out.NodeFailed) != 1 {
|
||||
t.Errorf("node failed = %+v", out.NodeFailed)
|
||||
}
|
||||
|
||||
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
|
||||
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
|
||||
three := m
|
||||
three.Nodes = []string{"ace", "g14", "novox"}
|
||||
failedHave := n.Haves["ace"]["mail"]
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
|
||||
n.Online["novox"] = true
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
|
||||
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if _, alone := out.NodeFailed["g14"]; !alone {
|
||||
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
|
||||
}
|
||||
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
|
||||
}
|
||||
if _, written := out.Wants["g14"]; written {
|
||||
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
|
||||
}
|
||||
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
delete(before, "novox")
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
|
||||
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
|
||||
}
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
|
||||
|
||||
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
|
||||
// not one node's failure (D3). Either way F is held and ace is set back.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
online map[string]bool
|
||||
novox delivery.Health
|
||||
}{
|
||||
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
|
||||
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
|
||||
} {
|
||||
n.Online = c.online
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
|
||||
if c.novox == delivery.HealthOK {
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}
|
||||
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
|
||||
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld {
|
||||
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4: a module holding data is not set back; its want stays and says why.
|
||||
@@ -427,6 +493,23 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
|
||||
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
|
||||
// condition stays raised. A node with a live want keeps it and is not on neither list.
|
||||
m.Composed = composedEverywhere
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||
if w, wrote := out.Wants["ace"]; wrote {
|
||||
t.Errorf("the bus bar wrote ace's want %+v", w)
|
||||
}
|
||||
h, on := out.Neither["ace"]
|
||||
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
|
||||
!strings.Contains(h.Words, "assigner_put") {
|
||||
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
|
||||
}
|
||||
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
|
||||
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
|
||||
}
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestManifest(t *testing.T) {
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.placement",
|
||||
"mesh-delivery-indexer.placement-candidates",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
@@ -31,7 +32,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
|
||||
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
@@ -166,7 +168,8 @@ func main() {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
|
||||
@@ -149,6 +149,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which builder_stuck says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
account, err := process.ReadAccount(path)
|
||||
@@ -162,7 +174,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
@@ -280,24 +292,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
}
|
||||
})
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
}
|
||||
})
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
select {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||
@@ -20,7 +21,8 @@ import (
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
@@ -315,6 +317,11 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
return step, step.Reason, errAgain
|
||||
}
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
|
||||
@@ -690,12 +690,14 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
}{
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
@@ -714,7 +716,11 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
if !c.byHand {
|
||||
dead := &buildrun.DeadLetters{}
|
||||
if c.dead {
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||
}
|
||||
if !c.byHand && !c.dead {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
@@ -724,6 +730,10 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.dead {
|
||||
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
|
||||
}
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
@@ -757,13 +767,16 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
|
||||
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
@@ -836,12 +849,17 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
|
||||
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
|
||||
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
|
||||
@@ -106,7 +106,9 @@ type StuckReader struct {
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
Now func() time.Time
|
||||
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
||||
Hearing StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
@@ -149,6 +151,20 @@ func parallel(n int, read func(i int) error) error {
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
||||
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
||||
func (r StuckReader) deadLetters() map[string]any {
|
||||
if r.Hearing.Get == nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
||||
}
|
||||
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
||||
}
|
||||
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
@@ -268,6 +284,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
answer["dead_letters"] = r.deadLetters()
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -53,10 +54,36 @@ func read(s stdio.KeptState) check.StateRead {
|
||||
}}
|
||||
}
|
||||
|
||||
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
|
||||
// (the commit the module's record was computed at); "" when the record is not readable.
|
||||
type runsOf struct{ modules check.StateRead }
|
||||
|
||||
func (r runsOf) CommitOf(module string) string {
|
||||
if r.modules.Get == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := r.modules.Get(module)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return m.Commit
|
||||
}
|
||||
|
||||
// stateStore is a state the tools write: the tries.
|
||||
type stateStore struct {
|
||||
check.StateRead
|
||||
create func(key string, value []byte) error
|
||||
entry func(key string) (json.RawMessage, uint64, error)
|
||||
update func(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
|
||||
func (s stateStore) Update(key string, value []byte, revision uint64) error {
|
||||
return s.update(key, value, revision)
|
||||
}
|
||||
|
||||
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
||||
@@ -66,19 +93,21 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
|
||||
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
||||
type Trying struct {
|
||||
Tries check.TryStore
|
||||
Ender check.TryEnder
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
|
||||
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
|
||||
func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
{Name: check.TryTool,
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
@@ -106,7 +135,8 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}},
|
||||
{Name: check.TryReadTool,
|
||||
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.",
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
|
||||
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -115,6 +145,28 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if tr.Ender == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be written here")
|
||||
}
|
||||
now := time.Now
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,7 +226,10 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
@@ -193,7 +248,9 @@ func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
trunks := func() (map[string]string, error) {
|
||||
@@ -205,13 +262,21 @@ func main() {
|
||||
return s.Heads, err
|
||||
}
|
||||
r.Trunks = trunks
|
||||
trying := Trying{
|
||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}, entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := tries.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := tries.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -53,13 +53,19 @@ func (m *memStore) Create(key string, value []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
|
||||
func (m *memStore) Update(key string, value []byte, revision uint64) error {
|
||||
m.vals[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
||||
store := &memStore{vals: map[string]json.RawMessage{}}
|
||||
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
|
||||
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
@@ -74,6 +80,14 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
t.Errorf("end = %v, %v", ended, err)
|
||||
}
|
||||
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
|
||||
t.Error("an ended try was ended again")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
||||
err.Error() != "no level of novox/hq is checked by the checker" {
|
||||
t.Errorf("hq: %v", err)
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -79,6 +80,8 @@ type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
conflict atomic.Bool
|
||||
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
|
||||
hold atomic.Value
|
||||
mu sync.Mutex
|
||||
calls map[string]int // by `<module>.<tool>`
|
||||
testTool *testing.T
|
||||
@@ -98,6 +101,9 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
||||
var v any
|
||||
switch {
|
||||
case strings.HasSuffix(tool, "_fetch"):
|
||||
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
|
||||
<-f.ctx.Done()
|
||||
}
|
||||
name := "go-fetch-clean"
|
||||
if f.conflict.Load() {
|
||||
name = "go-fetch-conflict"
|
||||
@@ -298,6 +304,10 @@ func startBed(t *testing.T) *bed {
|
||||
}
|
||||
b.tools[module] = kv
|
||||
}
|
||||
// What the controller keeps of the messages a consumer gave up on (issue 734).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The checker's stream of events, which the indexer and the forge bridge hear.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -305,7 +315,8 @@ func startBed(t *testing.T) *bed {
|
||||
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
|
||||
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
|
||||
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
|
||||
delivery.TurnsBucket: delivery.TurnsHistory} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -416,7 +427,8 @@ func (b *bed) start(t *testing.T, env map[string]string) {
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
// The settle window of a group's heads (check.DefaultSettle) is cut short: each bed's candidate is written whole.
|
||||
e := map[string]string{"MESH_BROKER_FILE": broker, "MESH_CHECKER_SETTLE": "10ms"}
|
||||
for k, v := range env {
|
||||
e[k] = v
|
||||
}
|
||||
@@ -507,9 +519,12 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
||||
}
|
||||
}
|
||||
|
||||
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
|
||||
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
|
||||
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
|
||||
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
|
||||
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
|
||||
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
|
||||
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
|
||||
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
|
||||
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
|
||||
b := startBed(t)
|
||||
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
b.start(t, nil)
|
||||
@@ -526,6 +541,42 @@ func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1
|
||||
if got := keptCount(t, b); got != kept {
|
||||
t.Errorf("%d kept keys, was %d", got, kept)
|
||||
}
|
||||
|
||||
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
|
||||
at := time.Now().UTC()
|
||||
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tk, _ := delivery.TurnsKey(repo)
|
||||
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond) // the checker hears the turn
|
||||
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
|
||||
time.Sleep(time.Second)
|
||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
|
||||
}
|
||||
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.verdict(t, third, ended)
|
||||
|
||||
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
|
||||
tests = b.fake.count("mesh-build-go.build_go_test")
|
||||
const moved = "abababababababababababababababababababab"
|
||||
b.setTrunk(t, map[string]string{repo: moved})
|
||||
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
|
||||
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
|
||||
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
|
||||
}
|
||||
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
|
||||
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
|
||||
}
|
||||
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
|
||||
t.Errorf("the carried status says %q", o.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func keptCount(t *testing.T, b *bed) int {
|
||||
@@ -676,6 +727,49 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
t.Errorf("a try wrote %s: %v", name, keys)
|
||||
}
|
||||
}
|
||||
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
|
||||
// message id) ends as error saying so, and its head asked again is a new try.
|
||||
b.fake.hold.Store(head2)
|
||||
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askID := ""
|
||||
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
keys, _ := steps.Keys(b.ctx)
|
||||
for _, k := range keys {
|
||||
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
|
||||
askID = strings.TrimPrefix(k, "n1.")
|
||||
}
|
||||
}
|
||||
}
|
||||
if askID == "" {
|
||||
t.Fatal("the held try's ask was never taken")
|
||||
}
|
||||
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
|
||||
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
|
||||
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
|
||||
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
|
||||
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
|
||||
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
raw, _ := store.Get(lost)
|
||||
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
|
||||
break
|
||||
}
|
||||
}
|
||||
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
|
||||
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
|
||||
}
|
||||
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
|
||||
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
|
||||
@@ -702,7 +796,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
||||
break
|
||||
}
|
||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
|
||||
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
|
||||
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
||||
if !equal(manifest, announced) {
|
||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||
}
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
||||
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
|
||||
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
|
||||
t.Errorf("readable %v, want %v", readable, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
|
||||
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
|
||||
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
|
||||
// and once again when it opens.
|
||||
type lazyTurns struct {
|
||||
turns *check.Turns
|
||||
open func() error
|
||||
logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
said bool
|
||||
}
|
||||
|
||||
// ensure opens the bucket once it is there; a failure is said once.
|
||||
func (l *lazyTurns) ensure() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.opened || l.open == nil {
|
||||
return
|
||||
}
|
||||
if err := l.open(); err != nil {
|
||||
if !l.said {
|
||||
l.said = true
|
||||
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
|
||||
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
l.opened = true
|
||||
if l.said {
|
||||
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
|
||||
}
|
||||
}
|
||||
|
||||
// Holding opens the bucket when it is not yet, then answers what the turns hold.
|
||||
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||
l.ensure()
|
||||
return l.turns.Holding(repositories, heads)
|
||||
}
|
||||
@@ -112,14 +112,17 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
|
||||
var keptStates = []announce.State{
|
||||
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
|
||||
{Name: "hearing"},
|
||||
{Name: "kept", Readable: true},
|
||||
{Name: "owed", Readable: true},
|
||||
{Name: "previews", Readable: true},
|
||||
{Name: "tries"},
|
||||
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
|
||||
{Name: "tries", Readable: true},
|
||||
{Name: "verdicts", Readable: true},
|
||||
}
|
||||
|
||||
@@ -186,6 +189,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
settleWindow, err := durationEnv(getenv, "MESH_CHECKER_SETTLE", check.DefaultSettle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
|
||||
if link == "" {
|
||||
link = defaultLink
|
||||
@@ -193,16 +200,32 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
tools := &buildrun.ToolchainSet{}
|
||||
presence := &buildrun.PresenceSet{}
|
||||
trunks := &check.Trunks{}
|
||||
turns := &check.Turns{}
|
||||
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
|
||||
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
||||
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which checker_status says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the checker (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
|
||||
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
|
||||
ch := &check.Checker{
|
||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||
Trunks: trunks,
|
||||
Turns: lazy,
|
||||
Bound: checkBound,
|
||||
Link: link,
|
||||
Logf: logger.Printf,
|
||||
Settle: settleWindow,
|
||||
}
|
||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||
// says nothing of the builds, until the checker starts again.
|
||||
@@ -228,7 +251,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return cw, nil
|
||||
}}
|
||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
|
||||
announce := func(ctx context.Context, event, key string) {
|
||||
body, _ := json.Marshal(map[string]string{"key": key})
|
||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||
@@ -236,8 +259,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
@@ -313,13 +337,42 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}()
|
||||
}
|
||||
}
|
||||
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
|
||||
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
|
||||
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
|
||||
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
|
||||
lazy.open = func() error {
|
||||
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
|
||||
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
lazy.ensure()
|
||||
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
||||
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
|
||||
before := repos.CommitOf(check.NodeEngineModule)
|
||||
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
|
||||
if after := repos.CommitOf(check.NodeEngineModule); after != before {
|
||||
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
|
||||
}
|
||||
})
|
||||
}()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 h1:MS7i4NVNQHMAMqRRFMdzbto31BHZY+BuEZY1ql3p4Rs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
|
||||
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
|
||||
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
|
||||
|
||||
// turnOf is a turn's value on a repository for a delivery of the heads hash.
|
||||
func turnOf(t testing.TB, repo, group, heads string) []byte {
|
||||
t.Helper()
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func turnKey(t testing.TB, repo string) string {
|
||||
t.Helper()
|
||||
k, err := delivery.TurnsKey(repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
|
||||
func waitFor(t testing.TB, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s did not happen", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
|
||||
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
|
||||
p := pullOf(repo, number, head)
|
||||
p.Modules = []string{"m"}
|
||||
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestOneCheckPerDelivery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for name, run := range map[string]func(t *testing.T){
|
||||
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
|
||||
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if j.count() != 0 || len(r.verdicts.vals) != 0 {
|
||||
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
|
||||
}
|
||||
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
|
||||
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
|
||||
}
|
||||
if !turns.Put(turnKey(t, repoA), nil, true) {
|
||||
t.Fatal("the turn's delete changed nothing")
|
||||
}
|
||||
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||
t.Errorf("judged %d times after the turn went, want once", j.count())
|
||||
}
|
||||
},
|
||||
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
c, _ := delivery.DecodeCandidate(cand)
|
||||
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 {
|
||||
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
|
||||
}
|
||||
},
|
||||
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
|
||||
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
|
||||
release, started := make(chan struct{}), make(chan struct{})
|
||||
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
<-started
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||
r.ch.Reconsider(ctx)
|
||||
close(release)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
|
||||
}
|
||||
},
|
||||
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
|
||||
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
|
||||
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
|
||||
var r *rig
|
||||
var kinds []string
|
||||
var from []map[string]string
|
||||
judged := make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||
if call == 1 {
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||
r.ch.TrunkMoved(ctx) // as the watch of the source says it
|
||||
close(judged)
|
||||
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
<-judged
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
|
||||
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
|
||||
}
|
||||
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
|
||||
}
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||
t.Errorf("the status after the carry is %q, want success", s)
|
||||
}
|
||||
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
|
||||
t.Error("the verdict key was made again: a carry rewrites it")
|
||||
}
|
||||
},
|
||||
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
|
||||
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
|
||||
var kept []bool
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||
kept = append(kept, c.Composition != nil)
|
||||
v := passing()
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
|
||||
return Outcome{Verdict: v}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
|
||||
r.ch.Wait()
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
|
||||
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
|
||||
}
|
||||
},
|
||||
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
|
||||
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
|
||||
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
|
||||
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
|
||||
var r *rig
|
||||
var from map[string]string
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 2 {
|
||||
from = c.CarriedFrom
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status during the sibling's carry is %q, want pending", s)
|
||||
}
|
||||
}
|
||||
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
|
||||
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
|
||||
}
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
|
||||
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
|
||||
}
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||
t.Errorf("the status after the carry is %q, want success", s)
|
||||
}
|
||||
},
|
||||
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
|
||||
// cleanly asks a whole check, not a carry.
|
||||
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
|
||||
var kinds []string
|
||||
var from []map[string]string
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||
if call == 2 {
|
||||
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
|
||||
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
|
||||
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
|
||||
}
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
|
||||
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
|
||||
}
|
||||
},
|
||||
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
|
||||
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
|
||||
"the line of a carry says where it was carried from": func(t *testing.T) {
|
||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||
v := passVerdict()
|
||||
v.Trunk = map[string]string{repoA: trunkA}
|
||||
v.Beside = map[string]string{catalogueRepository: trunkS2}
|
||||
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
|
||||
line := Line(v, cs, p, nil, nil, nil)
|
||||
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
|
||||
t.Errorf("the carried line = %q", line)
|
||||
}
|
||||
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
|
||||
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
|
||||
}
|
||||
cs.CarriedFrom = map[string]string{repoA: trunkA2}
|
||||
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
|
||||
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
|
||||
t.Errorf("the pending line of a carry = %q", pending)
|
||||
}
|
||||
},
|
||||
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
|
||||
"two merges waiting in one turn each get their carry": func(t *testing.T) {
|
||||
var r *rig
|
||||
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
|
||||
r = newRig(t, j)
|
||||
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
|
||||
r.ch.Candidate(ctx, "feat%2Fa", candA)
|
||||
r.ch.Candidate(ctx, "feat%2Fc", candC)
|
||||
r.ch.Wait()
|
||||
a, _ := delivery.DecodeCandidate(candA)
|
||||
c, _ := delivery.DecodeCandidate(candC)
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
|
||||
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
turns := &Turns{}
|
||||
turns.Put(turnKey(t, repoA), turn, false)
|
||||
r.ch.Turns = turns
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
for _, cand := range [][]byte{candA, candC} {
|
||||
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
|
||||
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
|
||||
}
|
||||
}
|
||||
},
|
||||
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
|
||||
"a new running node-engine is carried": func(t *testing.T) {
|
||||
runs := runsAt{nodeEngineModule: trunkS}
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
r.ch.Runs = runs
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
|
||||
r.ch.Wait()
|
||||
runs[nodeEngineModule] = trunkS2
|
||||
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
|
||||
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
|
||||
}
|
||||
},
|
||||
// §5: a carry that fails names the move it failed on.
|
||||
"a carry that fails names the move": func(t *testing.T) {
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}
|
||||
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
|
||||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
|
||||
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
|
||||
}
|
||||
},
|
||||
} {
|
||||
t.Run(name, run)
|
||||
}
|
||||
}
|
||||
|
||||
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
|
||||
func checkWordsOf(line string) (string, string) {
|
||||
for _, part := range strings.Split(line, " | ") {
|
||||
f := strings.Fields(part)
|
||||
for i := 0; i+2 < len(f); i++ {
|
||||
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
|
||||
return f[i], f[i+2]
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -96,9 +97,19 @@ func notJudged(v delivery.Verdict) bool {
|
||||
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
|
||||
}
|
||||
|
||||
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
|
||||
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
|
||||
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
|
||||
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
|
||||
func unanswered(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
|
||||
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
|
||||
}
|
||||
|
||||
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
|
||||
// assigner was not running (the assigner started since): the verdict and its status are written again with it. A
|
||||
// composition of heads no candidate holds now, or of a verdict that judged its composition, is left.
|
||||
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
|
||||
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
|
||||
// of a verdict that judged its composition otherwise, is left.
|
||||
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
|
||||
ch.init()
|
||||
if comp.State == delivery.CompositionWaits {
|
||||
@@ -123,10 +134,21 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
|
||||
return
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(cur)
|
||||
if err != nil || v.State != delivery.VerdictPass || !notJudged(v) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case v.State == delivery.VerdictPass && notJudged(v):
|
||||
case unanswered(v) && comp.Words != v.Composition.Words:
|
||||
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
|
||||
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
|
||||
default:
|
||||
return
|
||||
}
|
||||
withComposition(&v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = cs.FPSet
|
||||
}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err != nil {
|
||||
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
||||
|
||||
@@ -113,6 +113,27 @@ func TestTheAssignerStartingBringsTheComposition(t *testing.T) {
|
||||
t.Errorf("the status's line says %q", o.Text)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0559 §5: a verdict that failed only because a placed node had not answered takes the same heads' later
|
||||
// composition in on its event, a pass making it pass with no new check; a composition failing the same way is left.
|
||||
ch, cs := checkerWithNotJudged(t)
|
||||
silent := delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionFail, SDK: "x", Machine: "g14",
|
||||
Words: "g14 (offline) " + delivery.CompositionUnanswered}
|
||||
ch.Composed(context.Background(), silent)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
comp delivery.CompositionVerdict
|
||||
state string
|
||||
}{
|
||||
{"the same failure again", silent, delivery.VerdictFail},
|
||||
{"the node's answer", delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionPass, SDK: "x"}, delivery.VerdictPass},
|
||||
} {
|
||||
ch.Composed(context.Background(), row.comp)
|
||||
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
|
||||
if v, err := delivery.DecodeVerdict(raw); err != nil || v.State != row.state || v.Composition == nil || v.Composition.State != row.comp.State {
|
||||
t.Errorf("%s: the verdict is %+v, %v", row.name, v, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged.
|
||||
|
||||
@@ -12,8 +12,18 @@ type BesideRef struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
|
||||
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
|
||||
// group's member at its head do not move with a trunk.
|
||||
atTrunk bool
|
||||
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
|
||||
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
|
||||
runs bool
|
||||
}
|
||||
|
||||
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
|
||||
const NodeEngineModule = nodeEngineModule
|
||||
|
||||
const (
|
||||
catalogueRepository = "novox/mesh-catalog"
|
||||
nodeEngineRepo = "novox/mesh-host"
|
||||
@@ -51,18 +61,21 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
switch {
|
||||
case repo == catalogueRepository:
|
||||
if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
|
||||
}
|
||||
case repo == nodeEngineRepo:
|
||||
commit := ""
|
||||
if j.Runs != nil {
|
||||
commit = j.Runs.CommitOf(nodeEngineModule)
|
||||
}
|
||||
atTrunk := false
|
||||
if commit == "" {
|
||||
commit = heads[repo]
|
||||
commit, atTrunk = heads[repo], true
|
||||
}
|
||||
if commit != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: commit})
|
||||
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
|
||||
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
|
||||
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
|
||||
}
|
||||
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
|
||||
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
|
||||
@@ -82,7 +95,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
if member := c.member(repo); member != nil {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||
} else if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
|
||||
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
|
||||
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
|
||||
// checker that cannot place it says so, where placing none left that test to skip.
|
||||
@@ -108,3 +121,13 @@ func refOf(number int) string {
|
||||
}
|
||||
return "refs/pull/" + strconv.Itoa(number) + "/head"
|
||||
}
|
||||
|
||||
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
|
||||
// verdict names as Beside (ADR 0548 §6).
|
||||
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
|
||||
for _, r := range refs {
|
||||
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
|
||||
into[r.Repository] = r.Commit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A change that adds or modifies a compiled program fails (issue 1577): it is build output, `go build` in a module's folder
|
||||
// writes one beside go.mod, and `git add -A` takes it. The build module's fetch step, which holds the merged clone, reads the
|
||||
// first bytes of each changed file and reports the programs (mesh-sdk buildstep.MergeInto); the checker judges the answer.
|
||||
|
||||
// NoProgramsReported is the warning of a build module whose fetch answered no programs field: an older module, which cannot
|
||||
// tell, so the check does not refuse a program it did not look for.
|
||||
const NoProgramsReported = "the build module did not report programs: a compiled program in the change is not refused"
|
||||
|
||||
// checkBinariesOf fails a pull request whose change adds or modifies a compiled program, naming each file. It judges the
|
||||
// first fetch answer of the pull request, whichever build module gave it; a build module that reports no programs leaves a
|
||||
// warning on the check and passes it.
|
||||
func (st *state) checkBinariesOf(t *target) verdictPtr {
|
||||
key := fmt.Sprintf("%s#%d", t.pull.Repository, t.pull.Number)
|
||||
if t.fetched.Merge == "" {
|
||||
return nil // no merge was made: nothing was changed to judge
|
||||
}
|
||||
if t.fetched.Programs == nil {
|
||||
st.c.note(fmt.Sprintf("%s: %s (%s)", key, NoProgramsReported, t.module))
|
||||
st.c.warn(key, fmt.Sprintf("a compiled program is not refused: %s reported no programs", t.module))
|
||||
st.j.logf("%s: %s (%s)", key, NoProgramsReported, t.module)
|
||||
return nil
|
||||
}
|
||||
var found []string
|
||||
for _, p := range *t.fetched.Programs {
|
||||
if p.Commit != "" {
|
||||
found = append(found, fmt.Sprintf("%s: a compiled program added in %s and removed later: it stays in history; squash the branch (issue 1577)", p.Path, short12(p.Commit)))
|
||||
continue
|
||||
}
|
||||
found = append(found, p.Path+": a compiled program is build output: remove it and add it to .gitignore (issue 1577)")
|
||||
}
|
||||
if len(found) > 0 {
|
||||
return failed(t.node, fmt.Sprintf("%s: %s", key, strings.Join(found, "; ")))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestAChangeWhoseFetchReportsProgramsFailsNamingEachFile(t *testing.T) { // 1577
|
||||
prog := func(p ...string) *[]ProgramFile {
|
||||
out := []ProgramFile{}
|
||||
for _, f := range p {
|
||||
path, commit, _ := strings.Cut(f, "@") // `path@commit`: added by a commit of the branch and removed later
|
||||
out = append(out, ProgramFile{Path: path, Kind: "ELF", Commit: commit})
|
||||
}
|
||||
return &out
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
programs *[]ProgramFile
|
||||
fail []string // paths named; none passes
|
||||
warned bool // the build module reported no programs field
|
||||
}{
|
||||
{"none found", prog(), nil, false},
|
||||
{"one program", prog("modules/m/m"), []string{"modules/m/m"}, false},
|
||||
{"two programs", prog("a/x", "b/y"), []string{"a/x", "b/y"}, false},
|
||||
{"added and removed in the branch", prog("gone/prog@0123456789abcdef0123"), []string{"gone/prog: a compiled program added in 0123456789ab and removed later: it stays in history; squash the branch (issue 1577)"}, false},
|
||||
{"an older build module reports nothing", nil, nil, true},
|
||||
} {
|
||||
b := newBench(t)
|
||||
b.fetch.Merge, b.fetch.Programs = "mmmm", c.programs
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||
v := b.judge(t, cs).Verdict
|
||||
if got := slices.ContainsFunc(cs.Notes(), func(n string) bool { return strings.Contains(n, NoProgramsReported) }); got != c.warned {
|
||||
t.Errorf("%s: warned = %v, want %v (%v)", c.name, got, c.warned, cs.Notes())
|
||||
}
|
||||
line := Line(v, cs, cs.Pulls[0], nil, nil, nil)
|
||||
if got := strings.Contains(line, "a compiled program is not refused: mesh-build-go reported no programs"); got != c.warned {
|
||||
t.Errorf("%s: the status line holds the warning = %v, want %v: %s", c.name, got, c.warned, line)
|
||||
}
|
||||
if len(c.fail) == 0 {
|
||||
if v.State != delivery.VerdictPass {
|
||||
t.Errorf("%s: verdict = %+v", c.name, v)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if v.State != delivery.VerdictFail {
|
||||
t.Errorf("%s: verdict = %+v, want a failure", c.name, v)
|
||||
continue
|
||||
}
|
||||
for _, p := range c.fail {
|
||||
want := p
|
||||
if !strings.Contains(p, " added in ") {
|
||||
want = p + ": a compiled program is build output: remove it and add it to .gitignore (issue 1577)"
|
||||
}
|
||||
if !strings.Contains(v.Failed.Words, want) {
|
||||
t.Errorf("%s: words %q do not name %s", c.name, v.Failed.Words, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,12 +43,43 @@ type Case struct {
|
||||
Try *TryOptions
|
||||
|
||||
notesMu sync.Mutex
|
||||
notes []string
|
||||
// warnings are, by pull request, what part of its check did not run (Case.warn).
|
||||
warnings map[string]string
|
||||
notes []string
|
||||
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
|
||||
asks []string
|
||||
// Watch, when set, hears each ask the check puts, as it is put (step nil), and each value its step key takes while
|
||||
// the check waits for it: the node that took it, the phase it says, its end. A try writes them into its record, so a
|
||||
// running try names its asks and where each stands (issue 1890). Called from the check's goroutines; it must not block
|
||||
// for long.
|
||||
Watch func(askID string, step *delivery.Step)
|
||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||
Progress *Progress
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
|
||||
// seats' recheck stream, every other check on check (ADR 0536 §4).
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
|
||||
Kind string
|
||||
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
|
||||
// nil for a whole check.
|
||||
CarriedFrom map[string]string
|
||||
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
|
||||
FPSet string
|
||||
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
|
||||
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
|
||||
Composition *delivery.Composition
|
||||
}
|
||||
|
||||
// Asked records the id of an ask the check put.
|
||||
func (c *Case) Asked(id string) {
|
||||
c.notesMu.Lock()
|
||||
c.asks = append(c.asks, id)
|
||||
c.notesMu.Unlock()
|
||||
}
|
||||
|
||||
// Asks are the ids of the asks the check put, in order.
|
||||
func (c *Case) Asks() []string {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
// Progress is the step a check is at, written by the check and read when its bound passes.
|
||||
@@ -80,7 +111,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
|
||||
for _, p := range c.Pulls {
|
||||
switch head, known := trunk[p.Repository]; {
|
||||
case !known || head == "":
|
||||
@@ -162,6 +193,23 @@ func (c *Case) note(s string) {
|
||||
}
|
||||
}
|
||||
|
||||
// warn says of a pull request (pullKey) that a part of its check did not run, in words for its status line (issue 1577).
|
||||
func (c *Case) warn(key, words string) {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
if c.warnings == nil {
|
||||
c.warnings = map[string]string{}
|
||||
}
|
||||
c.warnings[key] = words
|
||||
}
|
||||
|
||||
// Warning is what warn said of a pull request, or "".
|
||||
func (c *Case) Warning(key string) string {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
return c.warnings[key]
|
||||
}
|
||||
|
||||
// Notes are the words noted on the case, in the order they were noted.
|
||||
func (c *Case) Notes() []string {
|
||||
c.notesMu.Lock()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -42,6 +43,9 @@ type Outcome struct {
|
||||
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
|
||||
// that changes none is not named.
|
||||
Judged map[string]string
|
||||
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
|
||||
// a move of one is carried.
|
||||
Beside map[string]string
|
||||
}
|
||||
|
||||
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
||||
@@ -65,6 +69,14 @@ type Checker struct {
|
||||
Owed KV
|
||||
Judge Judge
|
||||
Trunks interface{ Heads() map[string]string }
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
|
||||
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
|
||||
// waiting delivery's own heads; nil holds none.
|
||||
Turns interface {
|
||||
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
|
||||
}
|
||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||
Bound time.Duration
|
||||
// Link is the base of the address a status points at; the verdict key follows.
|
||||
@@ -73,6 +85,10 @@ type Checker struct {
|
||||
Announce Announcer
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// Settle is how long a group's heads must stay the same after the candidate last changed them before its check starts
|
||||
// (issue 1890): the members of a group are announced one after another, and a check of each partial set is thrown
|
||||
// away at the next. None when 0; DefaultSettle in the module.
|
||||
Settle time.Duration
|
||||
|
||||
once sync.Once
|
||||
wg sync.WaitGroup
|
||||
@@ -83,15 +99,78 @@ type Checker struct {
|
||||
tried map[string]bool // by verdict key: looked at since this process started
|
||||
epoch uint64 // changes of the nodes seen since this process started
|
||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||
held map[string]string // by verdict key: the turn that holds its check off, as last said
|
||||
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
|
||||
settling map[string]*window // by candidates key: the window its heads settle in
|
||||
}
|
||||
|
||||
// liveCheck is a check running in this process: the candidates key of its group and the end of its run.
|
||||
type liveCheck struct {
|
||||
group string
|
||||
end context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// window is a group's settle window: aborted with errUnsettled when its heads change again or the group goes, and with
|
||||
// errSettled to end it at once.
|
||||
type window struct{ abort context.CancelCauseFunc }
|
||||
|
||||
var (
|
||||
errUnsettled = errors.New("the group's heads changed again")
|
||||
errSettled = errors.New("the group's heads settled")
|
||||
)
|
||||
|
||||
// settleNow ends the settle window of every group at once, as if Settle had passed.
|
||||
func (ch *Checker) settleNow() {
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
for _, w := range ch.settling {
|
||||
w.abort(errSettled)
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultSettle is the module's Settle (issue 1890). On 2026-10-26 the three members of a group were announced within 2.8
|
||||
// seconds of each other (20:31:52.3, :53.7, :55.1 UTC), a session opening or pushing them one API call after another;
|
||||
// fifteen seconds holds five times that, and costs a check of 10 to 18 minutes under 2.5 % of its time. A member that comes
|
||||
// later still is no waste beyond its asks already taken: the check of the heads it replaced is ended at once.
|
||||
const DefaultSettle = 15 * time.Second
|
||||
|
||||
// EndedWords open the words of a verdict the checker ended without judging it, its heads no candidate's any more (issue
|
||||
// 1890): never a failure of the change, and judged whole if a candidate holds those heads again.
|
||||
const EndedWords = "ended: "
|
||||
|
||||
// IsEnded says whether a verdict was ended without being judged (EndedWords).
|
||||
func IsEnded(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, EndedWords)
|
||||
}
|
||||
|
||||
// superseded is the cause a running check is ended with: the heads that replaced its own, or none when its group went.
|
||||
type superseded struct{ by string }
|
||||
|
||||
func (s superseded) Error() string {
|
||||
if s.by == "" {
|
||||
return EndedWords + "no candidate holds these heads any more: the group was closed or merged"
|
||||
}
|
||||
return EndedWords + "superseded by " + s.by
|
||||
}
|
||||
|
||||
func (ch *Checker) init() {
|
||||
ch.once.Do(func() {
|
||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
ch.nodesFail = map[string]uint64{}
|
||||
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
|
||||
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
|
||||
})
|
||||
}
|
||||
|
||||
// endOthers ends every check of the group running in this process whose verdict key is not keep: no candidate holds its
|
||||
// heads any more (issue 1890). Called with ch.mu held.
|
||||
func (ch *Checker) endOthers(group, keep, by string) {
|
||||
for k, l := range ch.live {
|
||||
if l.group == group && k != keep {
|
||||
l.end(superseded{by: by})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ch *Checker) logf(format string, args ...any) {
|
||||
if ch.Logf != nil {
|
||||
ch.Logf(format, args...)
|
||||
@@ -124,17 +203,62 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
|
||||
ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
|
||||
return
|
||||
}
|
||||
vkey, _ := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
ch.mu.Lock()
|
||||
prev, had := ch.known[key]
|
||||
ch.known[key] = c
|
||||
// A check of heads this candidate no longer holds is ended at once (issue 1890): a member's new head, a member joining
|
||||
// or leaving.
|
||||
ch.endOthers(key, vkey, GateHeads(c))
|
||||
moved := !had || prev.HeadsHash != c.HeadsHash
|
||||
if ch.Settle > 0 && (moved || ch.settling[key] != nil) {
|
||||
// The group's heads changed: its check waits until they have been the same for Settle, so a group whose members
|
||||
// are announced one after another is checked once, whole. A later change starts the window again. The wait is a
|
||||
// context's bound, no timer (ADR 0437 §8).
|
||||
if w := ch.settling[key]; w != nil {
|
||||
w.abort(errUnsettled)
|
||||
}
|
||||
base, abort := context.WithCancelCause(ctx)
|
||||
w := &window{abort: abort}
|
||||
ch.settling[key] = w
|
||||
ch.wg.Add(1)
|
||||
go func() {
|
||||
defer ch.wg.Done()
|
||||
wait, done := context.WithTimeout(base, ch.Settle)
|
||||
defer done()
|
||||
<-wait.Done()
|
||||
settled := errors.Is(wait.Err(), context.DeadlineExceeded) || errors.Is(context.Cause(wait), errSettled)
|
||||
ch.mu.Lock()
|
||||
if !settled || ch.settling[key] != w {
|
||||
ch.mu.Unlock()
|
||||
return // changed again, forgotten, or the checker stops
|
||||
}
|
||||
delete(ch.settling, key)
|
||||
now, ok := ch.known[key]
|
||||
ch.mu.Unlock()
|
||||
abort(nil)
|
||||
if ok {
|
||||
ch.consider(ctx, key, now)
|
||||
}
|
||||
}()
|
||||
ch.mu.Unlock()
|
||||
return
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
ch.consider(ctx, key, c)
|
||||
}
|
||||
|
||||
// Forget takes a candidate out (its key was deleted).
|
||||
// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds
|
||||
// them any more.
|
||||
func (ch *Checker) Forget(key string) {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
delete(ch.known, key)
|
||||
if w := ch.settling[key]; w != nil {
|
||||
w.abort(errUnsettled)
|
||||
delete(ch.settling, key)
|
||||
}
|
||||
ch.endOthers(key, "", "")
|
||||
ch.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -177,32 +301,49 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
}
|
||||
cs.Queued = ch.now()
|
||||
ch.mu.Lock()
|
||||
if ch.settling[key] != nil {
|
||||
ch.mu.Unlock()
|
||||
return // its heads are settling: the window's end considers it (Settle)
|
||||
}
|
||||
if ch.inflight[cs.Key] {
|
||||
ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost
|
||||
ch.mu.Unlock()
|
||||
return
|
||||
}
|
||||
ch.inflight[cs.Key] = true
|
||||
run, end := context.WithCancelCause(ctx)
|
||||
ch.live[cs.Key] = liveCheck{group: key, end: end}
|
||||
ch.mu.Unlock()
|
||||
ch.wg.Add(1)
|
||||
go func() {
|
||||
defer ch.wg.Done()
|
||||
vkey := cs.Key
|
||||
defer func() {
|
||||
ch.mu.Lock()
|
||||
delete(ch.inflight, vkey)
|
||||
delete(ch.again, vkey)
|
||||
delete(ch.live, vkey)
|
||||
ch.mu.Unlock()
|
||||
end(nil)
|
||||
}()
|
||||
for {
|
||||
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
||||
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
||||
cs.Kind = ch.kindOf(ctx, cs)
|
||||
ch.try(ctx, key, cs)
|
||||
ch.prepare(ctx, cs)
|
||||
ch.try(ctx, run, key, cs)
|
||||
ch.mu.Lock()
|
||||
more := ch.again[cs.Key]
|
||||
delete(ch.again, cs.Key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(ch.inflight, cs.Key)
|
||||
ch.mu.Unlock()
|
||||
more := ch.again[vkey]
|
||||
delete(ch.again, vkey)
|
||||
ch.mu.Unlock()
|
||||
if !more || run.Err() != nil {
|
||||
return
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
// Looked at again for the same heads only: a candidate that moved to other heads has its own check (issue 1890).
|
||||
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
|
||||
if next.Key != vkey {
|
||||
return
|
||||
}
|
||||
cs = next
|
||||
}
|
||||
cs.Queued = ch.now()
|
||||
@@ -211,29 +352,132 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
}
|
||||
|
||||
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
||||
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
|
||||
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
|
||||
// §5), on the recheck stream, pulled after.
|
||||
const (
|
||||
KindNew = "new"
|
||||
KindNodes = "nodes"
|
||||
KindTrunk = "trunk"
|
||||
)
|
||||
|
||||
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
|
||||
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
|
||||
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
|
||||
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
|
||||
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
|
||||
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
|
||||
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
|
||||
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
|
||||
// to do.
|
||||
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
|
||||
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||
if err != nil || !found {
|
||||
return KindNew
|
||||
return
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
switch {
|
||||
case err != nil || v.State == delivery.VerdictRunning:
|
||||
return KindNew
|
||||
return
|
||||
case IsNodesFailure(v):
|
||||
return KindNodes
|
||||
cs.Kind = KindNodes
|
||||
return
|
||||
case IsConflictFailure(v):
|
||||
return // a head git refused: its next check is whole (ADR 0548 §7)
|
||||
case IsEnded(v):
|
||||
return // ended unjudged when its heads went (issue 1890): held again, they are checked whole
|
||||
}
|
||||
return KindTrunk
|
||||
cs.Kind = KindTrunk
|
||||
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
|
||||
cs.CarriedFrom = judgedOn(v)
|
||||
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
|
||||
kept := *c
|
||||
cs.Composition = &kept
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
|
||||
// read at its trunk's head (ADR 0548 §6).
|
||||
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
|
||||
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
|
||||
}
|
||||
|
||||
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
|
||||
// mesh runs when the indexer names one, as besideFor places them.
|
||||
func (ch *Checker) besideNow() map[string]string {
|
||||
return besideAt(ch.heads(), ch.Runs)
|
||||
}
|
||||
|
||||
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
|
||||
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
|
||||
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
|
||||
if runs != nil {
|
||||
if c := runs.CommitOf(nodeEngineModule); c != "" {
|
||||
out := make(map[string]string, len(now)+1)
|
||||
for r, h := range now {
|
||||
out[r] = h
|
||||
}
|
||||
out[nodeEngineRepo] = c
|
||||
return out
|
||||
}
|
||||
}
|
||||
return now
|
||||
}
|
||||
|
||||
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
|
||||
func besideMoved(beside, now map[string]string) bool {
|
||||
for r, c := range beside {
|
||||
if h := now[r]; h != "" && h != c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
|
||||
func judgedOn(v delivery.Verdict) map[string]string {
|
||||
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
|
||||
for r, c := range v.Beside {
|
||||
out[r] = c
|
||||
}
|
||||
for r, c := range v.Trunk {
|
||||
out[r] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
|
||||
// repositories the new verdict still judges or reads; nil when none of them moved.
|
||||
func carried(from map[string]string, v delivery.Verdict) map[string]string {
|
||||
out, moved := map[string]string{}, false
|
||||
for r, c := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if !ok {
|
||||
now, ok = v.Beside[r]
|
||||
}
|
||||
if ok && now != "" && c != "" {
|
||||
out[r] = c
|
||||
moved = moved || now != c
|
||||
}
|
||||
}
|
||||
if !moved {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
|
||||
// repository, sorted.
|
||||
func moveWords(from map[string]string, v delivery.Verdict) string {
|
||||
var moves []string
|
||||
for r, c := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if !ok {
|
||||
now = v.Beside[r]
|
||||
}
|
||||
if now != "" && now != c {
|
||||
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
|
||||
}
|
||||
}
|
||||
sort.Strings(moves)
|
||||
return strings.Join(moves, ", ")
|
||||
}
|
||||
|
||||
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
||||
@@ -265,6 +509,9 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return nil, fmt.Errorf("%s: %w", cs.Key, err)
|
||||
}
|
||||
if !found {
|
||||
if ch.heldByTurn(cs) {
|
||||
return nil, nil
|
||||
}
|
||||
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return nil, nil // another checker made it first
|
||||
@@ -285,15 +532,15 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return &claimed{rev}, nil
|
||||
}
|
||||
return nil, nil
|
||||
case !sameTrunk(cur.Trunk, cs.Trunk):
|
||||
retake = true
|
||||
case IsEnded(cur), ch.stale(cur, cs):
|
||||
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
|
||||
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
|
||||
ch.mu.Lock()
|
||||
at, ok := ch.nodesFail[cs.Key]
|
||||
ch.mu.Unlock()
|
||||
retake = !ok || epoch > at
|
||||
}
|
||||
if !retake {
|
||||
if !retake || ch.heldByTurn(cs) {
|
||||
return nil, nil
|
||||
}
|
||||
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
|
||||
@@ -306,6 +553,59 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return &claimed{rev}, nil
|
||||
}
|
||||
|
||||
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
|
||||
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
|
||||
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
|
||||
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
|
||||
func (ch *Checker) heldByTurn(cs *Case) bool {
|
||||
if ch.Turns == nil {
|
||||
return false
|
||||
}
|
||||
repos := make([]string, 0, len(cs.Trunk))
|
||||
for r := range cs.Trunk {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
|
||||
why := ""
|
||||
if held {
|
||||
why = repo + " " + w.Group + " " + w.Heads
|
||||
}
|
||||
ch.mu.Lock()
|
||||
said := ch.held[cs.Key]
|
||||
if why == "" {
|
||||
delete(ch.held, cs.Key)
|
||||
} else {
|
||||
ch.held[cs.Key] = why
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
if held && said != why {
|
||||
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
|
||||
cs.Key, w.Group, w.Heads, repo)
|
||||
}
|
||||
return held
|
||||
}
|
||||
|
||||
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
|
||||
func (ch *Checker) Held() map[string]string {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
out := make(map[string]string, len(ch.held))
|
||||
for k, v := range ch.held {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
|
||||
const ConflictWords = "does not merge into the trunk: files in conflict: "
|
||||
|
||||
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
|
||||
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
|
||||
func IsConflictFailure(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
|
||||
}
|
||||
|
||||
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
||||
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
||||
// seats' streams (ADR 0525 §6, issue 1912).
|
||||
@@ -313,9 +613,13 @@ func IsNodesFailure(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords)
|
||||
}
|
||||
|
||||
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
|
||||
// because ctx did is left running: the checker takes it again when it starts.
|
||||
func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
// try claims the check, judges it within its bound under run and writes the verdict and the owed statuses. A check that ends
|
||||
// because ctx did is left running: the checker takes it again when it starts. A check whose run was ended because no
|
||||
// candidate holds its heads any more (issue 1890) is ended: its verdict says so, and no status is written.
|
||||
func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
if run.Err() != nil {
|
||||
return // ended before it was claimed: nothing to write
|
||||
}
|
||||
ch.mu.Lock()
|
||||
first := !ch.tried[cs.Key]
|
||||
ch.tried[cs.Key] = true
|
||||
@@ -331,12 +635,47 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
}
|
||||
began := ch.now()
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
out := ch.judge(ctx, cs)
|
||||
out := ch.judge(run, cs)
|
||||
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
|
||||
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
|
||||
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
|
||||
if try >= LostAskTries {
|
||||
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
|
||||
" times and given up each time: " + UnheardWords)
|
||||
break
|
||||
}
|
||||
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
|
||||
out = ch.judge(run, cs)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var why superseded
|
||||
if errors.As(context.Cause(run), &why) {
|
||||
ch.end(ctx, cs, cl, why)
|
||||
return
|
||||
}
|
||||
v := out.Verdict
|
||||
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
||||
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
|
||||
for r, c := range out.Beside {
|
||||
if _, merged := cs.Trunk[r]; !merged && c != "" {
|
||||
if v.Beside == nil {
|
||||
v.Beside = map[string]string{}
|
||||
}
|
||||
v.Beside[r] = c
|
||||
}
|
||||
}
|
||||
if cs.CarriedFrom != nil {
|
||||
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
|
||||
switch v.State {
|
||||
case delivery.VerdictPass:
|
||||
v.CarriedFrom = carried(cs.CarriedFrom, v)
|
||||
case delivery.VerdictFail:
|
||||
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
|
||||
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
|
||||
}
|
||||
}
|
||||
}
|
||||
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
||||
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
|
||||
}
|
||||
@@ -358,17 +697,37 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
ch.nodesFail[cs.Key] = epoch
|
||||
ch.mu.Unlock()
|
||||
}
|
||||
if ch.movedSince(cs) {
|
||||
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
|
||||
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
||||
// check of the new trunk, which Reconsider starts, ends.
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
|
||||
ch.owePending(ctx, cs, v)
|
||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
|
||||
return
|
||||
}
|
||||
ch.writeOwed(ctx, cs, v, out)
|
||||
ch.logf("%s: %s", cs.Key, v.State)
|
||||
}
|
||||
|
||||
// end writes the verdict of a check whose heads no candidate holds any more (issue 1890), by compare-and-set on its claim,
|
||||
// and says it once in the log with the asks it had put: they stay in the build seats' streams, since an ask once put cannot
|
||||
// be taken back (buildrun.Asker has no withdraw), and are no longer waited for. No status is written: the heads that
|
||||
// replaced these have their own check, whose pending status stands.
|
||||
func (ch *Checker) end(ctx context.Context, cs *Case, cl *claimed, why superseded) {
|
||||
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Machine: "checker", Words: why.Error()}}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err == nil {
|
||||
_, err = ch.Verdicts.Update(ctx, cs.Key, data, cl.rev)
|
||||
}
|
||||
if err != nil {
|
||||
ch.logf("%s: %s, but the verdict could not say so: %v", cs.Key, why.Error(), err)
|
||||
return
|
||||
}
|
||||
stage, _ := cs.Progress.Now()
|
||||
asks := cs.Asks()
|
||||
ch.logf("%s: %s; it was at %q with %d asks put %v, which are no longer waited for", cs.Key, why.Error(), stage, len(asks), asks)
|
||||
}
|
||||
|
||||
// judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of
|
||||
// failure naming the machine and the step.
|
||||
func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome {
|
||||
@@ -467,7 +826,7 @@ func (ch *Checker) Sweep(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
cur.State = delivery.VerdictFail
|
||||
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
|
||||
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: EndedWords + "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
|
||||
data, err := delivery.EncodeVerdict(cur)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -502,14 +861,37 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
|
||||
if err != nil || !found {
|
||||
continue // never judged: no status to put out of date
|
||||
}
|
||||
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
if err == nil && !ch.stale(v, cs) {
|
||||
continue
|
||||
}
|
||||
if err == nil && v.State == delivery.VerdictPass {
|
||||
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
|
||||
}
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
ch.Reconsider(ctx)
|
||||
}
|
||||
|
||||
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
|
||||
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
|
||||
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
|
||||
next := judged
|
||||
ckey, err := delivery.CandidatesKey(v.Group)
|
||||
ch.mu.Lock()
|
||||
c, ok := ch.known[ckey]
|
||||
ch.mu.Unlock()
|
||||
if err == nil && ok {
|
||||
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
|
||||
next = now
|
||||
}
|
||||
}
|
||||
if v.State == delivery.VerdictPass {
|
||||
next.CarriedFrom = judgedOn(v)
|
||||
}
|
||||
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
|
||||
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
|
||||
func (ch *Checker) movedSince(cs *Case) bool {
|
||||
now := ch.heads()
|
||||
|
||||
@@ -3,6 +3,7 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -333,7 +334,10 @@ func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) {
|
||||
// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the
|
||||
// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again
|
||||
// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked.
|
||||
func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
|
||||
ctx, j := context.Background(), passJudge()
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
@@ -351,12 +355,91 @@ func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T)
|
||||
if err := r.ch.Sweep(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") {
|
||||
if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") {
|
||||
t.Errorf("the dead key = %+v", v)
|
||||
}
|
||||
if v := r.verdict(t, live); v.State != delivery.VerdictRunning {
|
||||
t.Errorf("the live key = %+v, want it left for the check to take over", v)
|
||||
}
|
||||
|
||||
// A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled.
|
||||
started, ended := make(chan struct{}), make(chan error, 1)
|
||||
j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
close(started)
|
||||
<-run.Done()
|
||||
ended <- context.Cause(run)
|
||||
return Outcome{}, run.Err()
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
var logged []string
|
||||
var logMu sync.Mutex
|
||||
r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() }
|
||||
alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", alone)
|
||||
<-started
|
||||
r.ch.Candidate(ctx, "feat%2Fx", both)
|
||||
var bothC delivery.Candidate
|
||||
_ = json.Unmarshal(both, &bothC)
|
||||
select {
|
||||
case err := <-ended:
|
||||
if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) {
|
||||
t.Errorf("the old check's run ended with %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the check of the heads the candidate left still runs")
|
||||
}
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) {
|
||||
t.Errorf("the superseded verdict = %+v", v.Failed)
|
||||
}
|
||||
if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != delivery.VerdictPass {
|
||||
t.Errorf("the group's verdict = %s", v.State)
|
||||
}
|
||||
okey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
|
||||
if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) {
|
||||
t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw)
|
||||
}
|
||||
logMu.Lock()
|
||||
said := 0
|
||||
for _, l := range logged {
|
||||
if strings.Contains(l, "ended: superseded") {
|
||||
said++
|
||||
}
|
||||
}
|
||||
logMu.Unlock()
|
||||
if said != 1 {
|
||||
t.Errorf("the end was said %d times: %v", said, logged)
|
||||
}
|
||||
// The member leaves again: the ended heads are held again and checked whole.
|
||||
r.ch.Candidate(ctx, "feat%2Fx", alone)
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, live); v.State != delivery.VerdictPass || j.count() != 3 {
|
||||
t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count())
|
||||
}
|
||||
|
||||
// Members announced one after another: only the settled, whole set is checked.
|
||||
j = passJudge()
|
||||
r = newRig(t, j)
|
||||
r.ch.Settle = time.Hour
|
||||
sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2))
|
||||
whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fy", sdk)
|
||||
r.ch.Candidate(ctx, "feat%2Fy", whole)
|
||||
r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing
|
||||
if j.count() != 0 {
|
||||
t.Fatalf("judged %d times before the heads settled", j.count())
|
||||
}
|
||||
r.ch.settleNow()
|
||||
r.ch.Wait()
|
||||
if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 {
|
||||
t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count())
|
||||
}
|
||||
if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != delivery.VerdictPass {
|
||||
t.Errorf("the whole set = %s", v.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
||||
b.ex.Compositions = f
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||
cs.FPSet = "fps-of-the-candidate"
|
||||
v := b.judge(t, cs).Verdict
|
||||
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
|
||||
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
|
||||
@@ -38,6 +39,15 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
if !compositionMentions(v, "passed") {
|
||||
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
|
||||
}
|
||||
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
|
||||
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
|
||||
}
|
||||
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
|
||||
carry := caseOf(t, pullOf(repoA, 7, head1))
|
||||
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
|
||||
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
|
||||
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
|
||||
}
|
||||
}
|
||||
|
||||
func compositionMentions(v delivery.Verdict, word string) bool {
|
||||
|
||||
@@ -116,6 +116,9 @@ type bench struct {
|
||||
fetch Fetched
|
||||
digest map[string]Digested // by level
|
||||
test func(c stepCall) (any, error)
|
||||
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
|
||||
// one (issue 890); an ask naming none runs on shanks, which then has none.
|
||||
declines int
|
||||
}
|
||||
|
||||
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
|
||||
@@ -151,8 +154,17 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
|
||||
if module == buildrun.GoBuildModule {
|
||||
steps.Digest = tool("digest")
|
||||
}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
if len(a.Needs) > 0 {
|
||||
if b.declines > 0 {
|
||||
b.declines--
|
||||
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
|
||||
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
|
||||
Reason: "declined", Result: result}
|
||||
}
|
||||
steps.Has = func(string) bool { return true }
|
||||
}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
@@ -276,6 +288,49 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
||||
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
||||
}
|
||||
|
||||
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
|
||||
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
|
||||
// the package alone, its kept passes not counted twice.
|
||||
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
|
||||
// placed ask that ran more than its unmet packages would run and count it again.
|
||||
b = newBench(t)
|
||||
b.declines = 1
|
||||
outside := 0
|
||||
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
|
||||
d := echoBeside(b, b.digest[c.args["level"].(string)])
|
||||
if d.Level == "integration" {
|
||||
outside++
|
||||
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
|
||||
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
|
||||
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
asks := b.asks.Asks()
|
||||
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
|
||||
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
|
||||
}
|
||||
for _, a := range asks[1:] {
|
||||
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
|
||||
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
|
||||
}
|
||||
}
|
||||
tests = b.steps.tool("test")
|
||||
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
|
||||
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
|
||||
}
|
||||
slow := captured[Tested](t, "go-test-slow")
|
||||
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
|
||||
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
|
||||
}
|
||||
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
|
||||
b.kept, b.declines = newMemKV(), 1000
|
||||
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
|
||||
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
|
||||
t.Errorf("with no node having the need: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
||||
@@ -620,9 +675,8 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
|
||||
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
|
||||
func echoBeside(b *bench, d Digested) Digested {
|
||||
var lines []string
|
||||
for _, f := range b.steps.tool("fetch") {
|
||||
refs, _ := refsOf(f.args["beside"]), true
|
||||
for _, r := range refs {
|
||||
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again)
|
||||
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) {
|
||||
lines = append(lines, r.Repository+" "+r.Commit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 19
|
||||
f := fatalAfterLog()
|
||||
f.Cause = ""
|
||||
v = levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "agreement_catalogue_test.go:130") {
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), "agreement_catalogue_test.go:130") {
|
||||
t.Errorf("words = %q", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
@@ -56,18 +56,45 @@ func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
|
||||
_ = delivery.VerdictFail
|
||||
}
|
||||
|
||||
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before.
|
||||
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3
|
||||
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before
|
||||
// (1901/AC3). A failure that hides its cause (none, or a passing line) is said loudly first in the words and in the status
|
||||
// line, naming the build module and the version it says it is, and the level still fails; a failure that keeps the
|
||||
// contract is worded as before (1901/AC8).
|
||||
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3, AC8
|
||||
old := `{"package":"p","test":"TestX","output":"x_test.go:1: logged\nx_test.go:2: the old kept lines"}`
|
||||
var f Failure
|
||||
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
|
||||
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
|
||||
}
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if v.Failed.Words != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
|
||||
if afterHid(v.Failed.Words) != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
|
||||
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
|
||||
}
|
||||
if got := firstFailed(*v); !strings.Contains(got, "logged") || strings.Contains(got, "\n") {
|
||||
t.Errorf("1901/AC3: status = %q", got)
|
||||
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.Contains(got, "\n") {
|
||||
t.Errorf("1901/AC3, AC8: status = %q", got)
|
||||
}
|
||||
|
||||
by := "mesh-build-node with mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78"
|
||||
for _, c := range []struct {
|
||||
name, module, by, cause, output, loud string
|
||||
}{
|
||||
{"no cause", "mesh-build-node", by, "", "test/b.test.ts: something", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): it sent no cause"},
|
||||
{"a passing line as its cause", "mesh-build-node", by, "\u2714 the same merge has the same id (8.006913ms)", "", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): its cause is a passing line"},
|
||||
{"from before the contract", "mesh-build-node", "", "", "\u2714 the same merge has the same id (8.006913ms)", "the build module hid the failure (mesh-build-node at a version before the failure contract): it sent no cause"},
|
||||
{"a real cause", "mesh-build-node", by, "one plus one is not three", "not ok 3 - fails", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
v := levelFailure(&target{module: c.module}, "unit", Tested{By: c.by, Failed: []Failure{{Package: "modules/gitea", Test: "fails", Cause: c.cause, Output: c.output}}})
|
||||
if v.State != delivery.VerdictFail {
|
||||
t.Errorf("1901/AC8: the verdict is %q, want it failed", v.State)
|
||||
}
|
||||
line := firstFailed(*v)
|
||||
switch {
|
||||
case c.loud == "" && strings.Contains(v.Failed.Words, "hid the failure"):
|
||||
t.Errorf("1901/AC8: a failure that keeps the contract is said hidden: %q", v.Failed.Words)
|
||||
case c.loud != "" && (!strings.HasPrefix(v.Failed.Words, c.loud) || !strings.Contains(line, c.loud) || !strings.Contains(v.Failures[0].Words, "hid the failure")):
|
||||
t.Errorf("1901/AC8: words %q, status %q, want both to start with %q", v.Failed.Words, line, c.loud)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -88,29 +87,3 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
|
||||
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
|
||||
ctx := context.Background()
|
||||
var r *rig
|
||||
release, judged := make(chan struct{}), make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||
defer close(judged)
|
||||
} else {
|
||||
<-release // the check asked again for the new trunk waits in the stream meanwhile
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
||||
<-judged
|
||||
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
|
||||
}
|
||||
close(release)
|
||||
r.ch.Wait()
|
||||
}
|
||||
|
||||
@@ -80,6 +80,9 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
|
||||
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
||||
type Examiner struct {
|
||||
Asks Asks
|
||||
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
||||
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
|
||||
DeclineBound time.Duration
|
||||
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
||||
Toolchains ToolchainEntries
|
||||
Kept Kept
|
||||
@@ -203,12 +206,15 @@ type state struct {
|
||||
nodes map[string]bool
|
||||
clone time.Duration
|
||||
slow []string
|
||||
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
|
||||
beside map[string]string
|
||||
}
|
||||
|
||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||
// be judged at all.
|
||||
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
|
||||
beside: map[string]string{}}
|
||||
if v := st.judgeAll(ctx); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
@@ -230,6 +236,13 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
||||
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
||||
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
||||
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
if st.c.Composition != nil && st.c.Try == nil {
|
||||
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
|
||||
kept := *st.c.Composition
|
||||
v.Composition = &kept
|
||||
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil || st.c.Try != nil {
|
||||
return
|
||||
}
|
||||
@@ -243,6 +256,9 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
return
|
||||
}
|
||||
withComposition(v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
|
||||
}
|
||||
}
|
||||
|
||||
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
||||
@@ -282,6 +298,9 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
|
||||
}
|
||||
}
|
||||
out := Outcome{Verdict: v, Slowest: slow}
|
||||
if len(st.beside) > 0 {
|
||||
out.Beside = st.beside
|
||||
}
|
||||
for _, t := range st.targets {
|
||||
if t.judged != "" {
|
||||
if out.Judged == nil {
|
||||
@@ -343,6 +362,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
t.beside = besideLines(beside)
|
||||
besideAtTrunk(st.c, beside, st.beside)
|
||||
}
|
||||
if p.Repository == catalogueRepository {
|
||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||
@@ -364,11 +384,25 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
||||
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
|
||||
const DefaultDeclineBound = 30 * time.Minute
|
||||
|
||||
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
|
||||
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
|
||||
// asked again of a node that has it (placed).
|
||||
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
|
||||
}
|
||||
|
||||
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
|
||||
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
|
||||
// is run or counted again (ADR 0552 §4).
|
||||
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
||||
p := t.pull
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
||||
placed := len(needs) > 0
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||
if t.runner != "" {
|
||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||
}
|
||||
@@ -380,17 +414,53 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||
}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Check: job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
At: st.j.now().UTC(), Check: job, Needs: needs}
|
||||
var step delivery.Step
|
||||
var firstDecline time.Time
|
||||
for {
|
||||
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
|
||||
if placed {
|
||||
what += " for a node with " + strings.Join(needs, ", ")
|
||||
}
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||
st.c.Progress.Set(what, "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
waiting := ctx
|
||||
if watch := st.c.Watch; watch != nil {
|
||||
id := a.ID
|
||||
watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
|
||||
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
|
||||
}
|
||||
var err error
|
||||
step, err = st.j.Asks.Wait(waiting, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
}
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
d, declined := delivery.DeclinedOf(step)
|
||||
if !declined {
|
||||
break
|
||||
}
|
||||
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
|
||||
// node that declined looks at the stream again only after its next pull's wait.
|
||||
now := st.j.now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
bound := st.j.DeclineBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultDeclineBound
|
||||
}
|
||||
if now.Sub(firstDecline) > bound {
|
||||
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
||||
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
||||
}
|
||||
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
||||
a.ID, a.At = buildstep.NewID(), now.UTC()
|
||||
}
|
||||
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
||||
took := time.Duration(0)
|
||||
@@ -414,18 +484,31 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
if st.c.Try == nil {
|
||||
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
}
|
||||
t.changed = t.fetched.Changed
|
||||
if !changesBuildGo(p.Repository, t.changed) {
|
||||
t.runner = "" // the build module used no runner: the merge does not change it
|
||||
}
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||
return v
|
||||
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||
return v
|
||||
}
|
||||
if v := st.checkBinariesOf(t); v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
ran := 0.0
|
||||
for _, la := range answer.Levels {
|
||||
if placed {
|
||||
if len(la.Unmet) > 0 {
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
|
||||
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
|
||||
}
|
||||
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
|
||||
}
|
||||
v, elapsed := st.level(ctx, t, la)
|
||||
ran += elapsed
|
||||
if v != nil {
|
||||
@@ -434,9 +517,48 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
}
|
||||
}
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
||||
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
|
||||
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
||||
var again, packages []string
|
||||
seen := map[string]bool{}
|
||||
for _, la := range answer.Levels {
|
||||
if len(again) > 0 || len(la.Unmet) > 0 {
|
||||
again = append(again, la.Level)
|
||||
}
|
||||
for _, u := range la.Unmet {
|
||||
if !seen[u.Package] {
|
||||
seen[u.Package] = true
|
||||
packages = append(packages, u.Package)
|
||||
}
|
||||
}
|
||||
}
|
||||
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
||||
return st.askNeeding(ctx, t, merged, again, unmet, packages)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unmetNames names the unmet packages and their needs.
|
||||
func unmetNames(unmet []delivery.PackageNeeds) string {
|
||||
var out []string
|
||||
for _, u := range unmet {
|
||||
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// unmetOf names the unmet packages of every level.
|
||||
func unmetOf(levels []delivery.LevelAnswer) string {
|
||||
var out []string
|
||||
for _, la := range levels {
|
||||
if len(la.Unmet) > 0 {
|
||||
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
||||
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
||||
@@ -541,6 +663,12 @@ const MaxWords = 4096
|
||||
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
|
||||
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
|
||||
func failureWords(f Failure) string {
|
||||
return failureWordsAfter(f, "")
|
||||
}
|
||||
|
||||
// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
|
||||
// the failure (that the build module hid it), which the status line then names.
|
||||
func failureWordsAfter(f Failure, loud string) string {
|
||||
out, passed := dropSummary(strings.TrimSpace(f.Output))
|
||||
cause := strings.TrimSpace(f.Cause)
|
||||
if cause == "" {
|
||||
@@ -556,11 +684,14 @@ func failureWords(f Failure) string {
|
||||
if len(cause) > MaxCause {
|
||||
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
|
||||
}
|
||||
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
|
||||
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
|
||||
budget := MaxWords
|
||||
if cause != "" {
|
||||
budget -= len(cause) + 1
|
||||
}
|
||||
if loud != "" {
|
||||
budget -= len(loud) + 1
|
||||
}
|
||||
if len(out) > budget {
|
||||
const marker = "... "
|
||||
tail := out[len(out)-(budget-len(marker)):]
|
||||
@@ -573,9 +704,31 @@ func failureWords(f Failure) string {
|
||||
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
|
||||
out = strings.TrimRight(cause+"\n"+out, "\n")
|
||||
}
|
||||
if loud != "" {
|
||||
out = strings.TrimRight(loud+"\n"+out, "\n")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
|
||||
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
|
||||
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
|
||||
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
|
||||
func hidWords(module, by string, f Failure) string {
|
||||
why := buildstep.Hidden(f)
|
||||
if why == "" {
|
||||
return ""
|
||||
}
|
||||
version := "a version before the failure contract"
|
||||
if module == "" {
|
||||
module = "an unnamed build module"
|
||||
}
|
||||
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
|
||||
module, version = m, v
|
||||
}
|
||||
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
|
||||
}
|
||||
|
||||
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
|
||||
// push the words past MaxWords).
|
||||
const MaxCause = 1024
|
||||
@@ -662,7 +815,7 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
|
||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||
name = res.Module
|
||||
}
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
|
||||
}
|
||||
}
|
||||
if len(all) == 0 { // no result: a level said failed with nothing to name
|
||||
|
||||
@@ -56,12 +56,22 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
|
||||
|
||||
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
|
||||
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
|
||||
// 1): it runs every check. It answers the packages it kept now.
|
||||
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a
|
||||
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
|
||||
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
|
||||
// packages it kept now.
|
||||
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
|
||||
var kept []string
|
||||
perPackage := false
|
||||
for _, p := range res.Packages {
|
||||
perPackage = perPackage || p.Skipped > 0
|
||||
}
|
||||
if res.Skipped > 0 && !perPackage {
|
||||
return kept, nil // an older build module: the skip cannot be placed in a package
|
||||
}
|
||||
for _, p := range res.Packages {
|
||||
digest, ok := digests[p.Package]
|
||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
|
||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 {
|
||||
continue
|
||||
}
|
||||
if failedIn(res, p.Package) {
|
||||
|
||||
@@ -74,6 +74,32 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
|
||||
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
|
||||
if err != nil || len(kept) != 0 {
|
||||
t.Fatalf("kept %v, %v, want nothing", kept, err)
|
||||
}
|
||||
if keys, _ := st.Keys(ctx); len(keys) != 0 {
|
||||
t.Errorf("the bucket holds %v, want none", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
|
||||
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
|
||||
{Package: "m/b", Passed: true, Tests: 3},
|
||||
}}
|
||||
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
|
||||
if err != nil || strings.Join(kept, ",") != "m/b" {
|
||||
t.Fatalf("kept %v, %v, want only m/b", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
|
||||
@@ -85,18 +85,22 @@ type pieces struct {
|
||||
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
||||
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
|
||||
judged string
|
||||
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
|
||||
carried string
|
||||
// warning says that a part of the check did not run, for instance that an older build module could not refuse a compiled program; never dropped.
|
||||
warning string
|
||||
}
|
||||
|
||||
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
|
||||
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
|
||||
|
||||
func (p pieces) join(drop int) string {
|
||||
parts := []string{p.head, p.commits, p.counts, p.judged}
|
||||
parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged}
|
||||
switch {
|
||||
case drop < 3 && p.times != "":
|
||||
parts = append(parts, p.times)
|
||||
case drop == 3 && p.shortTimes != "":
|
||||
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
|
||||
parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.shortTimes}
|
||||
}
|
||||
if p.over != "" {
|
||||
parts = append(parts, p.over)
|
||||
@@ -171,16 +175,62 @@ func buildWords(builds []BuildNote) string {
|
||||
return "builds: " + strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
|
||||
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
|
||||
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
|
||||
var own string
|
||||
var others []string
|
||||
for r, old := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if cs != nil && !ok {
|
||||
now, ok = cs.Trunk[r]
|
||||
}
|
||||
if !ok {
|
||||
now = v.Beside[r]
|
||||
}
|
||||
if now == "" || now == old {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(r, p.Repository) {
|
||||
own = short12(old)
|
||||
continue
|
||||
}
|
||||
name := r
|
||||
if i := strings.LastIndex(r, "/"); i >= 0 {
|
||||
name = r[i+1:]
|
||||
}
|
||||
others = append(others, name+" "+short12(old))
|
||||
}
|
||||
slices.Sort(others)
|
||||
if own != "" {
|
||||
others = append([]string{own}, others...)
|
||||
}
|
||||
return strings.Join(others, ", ")
|
||||
}
|
||||
|
||||
// parts makes the pieces of the line of a pull request of a case.
|
||||
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
|
||||
var pc pieces
|
||||
pc.judged = judged[pullKey(p)]
|
||||
if cs != nil {
|
||||
pc.warning = cs.Warning(pullKey(p))
|
||||
}
|
||||
commits := short12(p.Head) + " on " + short12(p.Trunk)
|
||||
switch v.State {
|
||||
case delivery.VerdictRunning:
|
||||
pc.head = "checking " + commits
|
||||
if cs != nil && cs.CarriedFrom != nil {
|
||||
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
|
||||
pc.head = "being carried to " + commits
|
||||
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
|
||||
pc.carried = "from " + w
|
||||
}
|
||||
}
|
||||
case delivery.VerdictPass:
|
||||
pc.head = "pass " + commits
|
||||
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
|
||||
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
|
||||
}
|
||||
default:
|
||||
pc.head = "failed: " + firstFailed(v)
|
||||
if n := failedCount(v); n > 1 {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
|
||||
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
|
||||
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
|
||||
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
|
||||
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
|
||||
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
|
||||
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
|
||||
|
||||
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
|
||||
// judged again when the nodes change.
|
||||
const LostAskTries = 3
|
||||
|
||||
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
|
||||
func IsLostAsk(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
|
||||
}
|
||||
|
||||
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
|
||||
func IsNotTaken(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
|
||||
}
|
||||
|
||||
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
|
||||
// buildrun.HearingKey), which checker_status says (dead_letters).
|
||||
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
|
||||
@@ -0,0 +1,170 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
|
||||
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
|
||||
// gives up (an ask no node takes).
|
||||
type losingAsks struct {
|
||||
*buildrun.FakeAsks
|
||||
answer func(delivery.StepAsk) delivery.Step
|
||||
dead *buildrun.DeadLetters
|
||||
never bool
|
||||
|
||||
mu sync.Mutex
|
||||
lose int
|
||||
}
|
||||
|
||||
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
||||
return err
|
||||
}
|
||||
l.mu.Lock()
|
||||
lost := l.lose > 0
|
||||
if lost {
|
||||
l.lose--
|
||||
}
|
||||
l.mu.Unlock()
|
||||
switch {
|
||||
case l.never:
|
||||
case lost:
|
||||
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
|
||||
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
|
||||
g, ok := buildrun.GivenUpOf(h)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
go l.dead.Heard(g)
|
||||
default:
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.Write(l.answer(a))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func losing(b *bench, lose int) *losingAsks {
|
||||
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
|
||||
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
|
||||
return l
|
||||
}
|
||||
|
||||
// endable is the tries state as the end tool sees it.
|
||||
type endable struct{ kv *memKV }
|
||||
|
||||
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
|
||||
v, rev, _, err := e.kv.Get(context.Background(), key)
|
||||
return v, rev, err
|
||||
}
|
||||
func (e endable) Update(key string, value []byte, revision uint64) error {
|
||||
_, err := e.kv.Update(context.Background(), key, value, revision)
|
||||
return err
|
||||
}
|
||||
|
||||
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
|
||||
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
|
||||
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
lose int
|
||||
never bool
|
||||
want string
|
||||
}{
|
||||
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
|
||||
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
l.never = row.never
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
got := readTry(t, kv, id)
|
||||
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
|
||||
t.Fatalf("try = %+v, want error saying %q", got, row.want)
|
||||
}
|
||||
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
|
||||
if err != nil || again == id {
|
||||
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("ended by hand", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
b := newBench(t)
|
||||
losing(b, 0).never = true
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
|
||||
t.Error("an end without a why was taken")
|
||||
}
|
||||
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
|
||||
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
|
||||
t.Fatalf("ended = %+v, %v", ended, err)
|
||||
}
|
||||
raw, _, _, _ = kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
|
||||
tr.Wait() // returns: the try's judging stopped
|
||||
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
|
||||
t.Errorf("try = %+v, want the hand's end kept", got)
|
||||
}
|
||||
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
|
||||
t.Errorf("a second end answered %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
|
||||
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
|
||||
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
lose int
|
||||
state string
|
||||
asks int
|
||||
}{
|
||||
{1, delivery.VerdictPass, 2},
|
||||
{LostAskTries, delivery.VerdictFail, LostAskTries},
|
||||
} {
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
r := newRig(t, b.ex)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != row.state || len(l.Asks()) != row.asks {
|
||||
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
|
||||
}
|
||||
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -38,11 +38,11 @@ func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
|
||||
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(v.Failed.Words, c.want) {
|
||||
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), c.want) {
|
||||
t.Errorf("1901/AC5: words after the loud line start %q, want %q", firstLine(afterHid(v.Failed.Words)), c.want)
|
||||
}
|
||||
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
|
||||
t.Errorf("1901/AC5: status = %q", got)
|
||||
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.HasSuffix(got, ": PASS") {
|
||||
t.Errorf("1901/AC5, AC8: status = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -63,7 +63,7 @@ func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
|
||||
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
|
||||
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), "the package failed with no test to name and no panic, signal or build error in its output") {
|
||||
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,7 @@ func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901
|
||||
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
|
||||
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
|
||||
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
@@ -90,3 +90,13 @@ func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// afterHid is a verdict's words after the checker's loud line, which a failure with no cause from an older build module
|
||||
// now starts with (1901/AC8).
|
||||
func afterHid(w string) string {
|
||||
if strings.HasPrefix(w, "the build module hid the failure") {
|
||||
_, rest, _ := strings.Cut(w, "\n")
|
||||
return rest
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
@@ -240,7 +240,12 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
||||
b.ex.Forge = forge
|
||||
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
||||
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
|
||||
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
|
||||
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
|
||||
}
|
||||
got := besideOf(t, b, controller)
|
||||
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
|
||||
@@ -324,7 +329,11 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
|
||||
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
||||
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
|
||||
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
|
||||
}
|
||||
got := besideOf(t, b, catalogueRepository)
|
||||
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
||||
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
|
||||
|
||||
@@ -75,3 +75,40 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
|
||||
t.Errorf("waiting = %v, want %v", st.Waiting, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 568 note 21: a verdict that read the node-engine at its running commit is not waiting for a trunk move while that commit
|
||||
// is still the one the mesh runs, though the node-engine's trunk has moved on (ADR 0548 §6).
|
||||
func TestCheckerStatusReadsTheNodeEngineAtItsRunningCommit(t *testing.T) { // 568/21
|
||||
cand := candidateOf(t, group(1), pullOf(repoA, 1, head1))
|
||||
ckey, _ := delivery.CandidatesKey(group(1))
|
||||
vkey := keyOf(t, group(1), cand)
|
||||
dc, _ := delivery.DecodeCandidate(cand)
|
||||
judged, err := delivery.EncodeVerdict(delivery.Verdict{Group: group(1), Heads: dc.HeadsHash, Trunk: map[string]string{repoA: trunkA}, Beside: map[string]string{nodeEngineRepo: "run111"}, State: delivery.VerdictPass, Levels: passing().Levels})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heads := func() (map[string]string, error) {
|
||||
return map[string]string{repoA: trunkA, nodeEngineRepo: "trunk222"}, nil
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
runs runsAt
|
||||
want int
|
||||
}{
|
||||
{"the running commit the verdict read", runsAt{nodeEngineModule: "run111"}, 0},
|
||||
{"the running commit moved", runsAt{nodeEngineModule: "run999"}, 1},
|
||||
{"no running commit known: the trunk's", nil, 1},
|
||||
} {
|
||||
rd := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{vkey: judged}.read(), Trunks: heads}
|
||||
if row.runs != nil {
|
||||
rd.Runs = row.runs
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(st.Waiting[KindTrunk]); got != row.want {
|
||||
t.Errorf("%s: waiting for a trunk move = %v, want %d", row.name, st.Waiting[KindTrunk], row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||
@@ -39,7 +40,14 @@ type Reader struct {
|
||||
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
||||
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
|
||||
TurnState StateRead
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
|
||||
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
||||
HearingState StateRead
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||
@@ -312,7 +320,68 @@ type Status struct {
|
||||
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
||||
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
||||
Waiting map[string][]string `json:"waiting_by_kind"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
|
||||
// the group of the merge that waits. Their check is claimed when that turn goes.
|
||||
HeldByTurn map[string]string `json:"held_by_turn"`
|
||||
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
||||
// whose ask is given up waits until its bound.
|
||||
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
|
||||
type DeadLettersHeard struct {
|
||||
Heard bool `json:"heard"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
|
||||
func (r Reader) deadLetters() DeadLettersHeard {
|
||||
if r.HearingState.Get == nil {
|
||||
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
|
||||
}
|
||||
raw, err := r.HearingState.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
|
||||
}
|
||||
|
||||
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
|
||||
func (r Reader) turns(st *Status) *Turns {
|
||||
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
|
||||
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
|
||||
return nil
|
||||
}
|
||||
keys, err := r.TurnState.Keys()
|
||||
if err != nil {
|
||||
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
|
||||
return nil
|
||||
}
|
||||
t := &Turns{}
|
||||
for _, k := range keys {
|
||||
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
|
||||
t.Put(k, raw, false)
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// heldBy says the turn that holds a candidate's check off, or "".
|
||||
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
|
||||
cs := CaseOf(c, heads)
|
||||
if t == nil || cs == nil {
|
||||
return ""
|
||||
}
|
||||
repos := make([]string, 0, len(cs.Trunk))
|
||||
for r := range cs.Trunk {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
|
||||
return repo + ": the approved merge of " + w.Group + " waits on it"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||
@@ -383,6 +452,8 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
st.Unjudged = []string{}
|
||||
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
||||
st.HeldByTurn = map[string]string{}
|
||||
turns := r.turns(&st)
|
||||
var heads map[string]string
|
||||
if r.Trunks == nil {
|
||||
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
||||
@@ -435,6 +506,9 @@ func (r Reader) Status() (Status, error) {
|
||||
if len(raw) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
if why := heldBy(turns, c, heads); why != "" {
|
||||
st.HeldByTurn[c.Group] = why
|
||||
}
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(raw)
|
||||
@@ -443,12 +517,16 @@ func (r Reader) Status() (Status, error) {
|
||||
case IsNodesFailure(v):
|
||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||
case heads != nil:
|
||||
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
|
||||
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
|
||||
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
||||
if why := heldBy(turns, c, heads); why != "" {
|
||||
st.HeldByTurn[c.Group] = why
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
st.DeadLetters = r.deadLetters()
|
||||
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
||||
return st, nil
|
||||
}
|
||||
|
||||
@@ -144,7 +144,10 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
CandidateState: fakeState{ckey: cand}.read(),
|
||||
VerdictState: fakeState{}.read(),
|
||||
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
|
||||
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
|
||||
`"at":"2026-10-26T13:00:00Z"}`)}.read(),
|
||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
@@ -156,9 +159,13 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||
t.Errorf("unjudged = %v", st.Unjudged)
|
||||
}
|
||||
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
|
||||
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
|
||||
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
|
||||
}
|
||||
// What cannot be read is said, not guessed.
|
||||
st, err = Reader{}.Status()
|
||||
if err != nil || len(st.Notes) == 0 {
|
||||
if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard {
|
||||
t.Errorf("empty status = %+v, %v", st, err)
|
||||
}
|
||||
}
|
||||
@@ -234,6 +241,18 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||
}
|
||||
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
|
||||
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
hkey, _ := delivery.CandidatesKey("feat/x")
|
||||
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
|
||||
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
|
||||
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
|
||||
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||
|
||||
@@ -3,6 +3,9 @@ package check
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
|
||||
@@ -42,6 +45,17 @@ type Fetched struct {
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]MergeTree `json:"trees,omitempty"`
|
||||
// Programs are the changed files whose first bytes are an executable binary, read by the build module in its workspace
|
||||
// (issue 1577); nil for a build module that does not report it, a pointer to an empty list for one that found none.
|
||||
Programs *[]ProgramFile `json:"programs,omitempty"`
|
||||
}
|
||||
|
||||
// ProgramFile is a changed file that is a compiled program, and its format (ELF, Mach-O or PE).
|
||||
type ProgramFile struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
// Commit is set for a program a commit of the branch added and a later one removed: it stays in history once merged.
|
||||
Commit string `json:"commit,omitempty"`
|
||||
}
|
||||
|
||||
// GoPackage is one package of a Go test result.
|
||||
@@ -52,16 +66,13 @@ type GoPackage struct {
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Tests int `json:"tests"`
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
// Skipped is how many of the package's tests skipped; an older build module names none.
|
||||
Skipped int `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// Failure is a test that failed, or a package that failed with no test to name.
|
||||
type Failure struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
|
||||
Cause string `json:"cause,omitempty"`
|
||||
}
|
||||
// Failure is a test that failed, or a package that failed with no test to name: the build modules' failure contract
|
||||
// (buildstep.Failure, issue 1901), its cause never empty and never a passing line from a build module that keeps it.
|
||||
type Failure = buildstep.Failure
|
||||
|
||||
// TestTime is one test and the seconds it took.
|
||||
type TestTime struct {
|
||||
@@ -94,6 +105,9 @@ type Tested struct {
|
||||
// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
|
||||
// workspace; absent from a build module that does not take the runner.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// By is the build module that answered and the SDK it was built with (buildstep.By); absent from a build module from
|
||||
// before the failure contract.
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
|
||||
@@ -107,4 +121,6 @@ type Digested struct {
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// Needs are the packages whose tests need something of the node at this level (issue 890).
|
||||
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ TestFetchWithABaseMergesAndAnswersThePreview and TestFetchWithAConflictEndsPasse
|
||||
into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d5804927 (the git commands are the test's fake,
|
||||
so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the
|
||||
FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview
|
||||
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679.
|
||||
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. Both clean fetches carry `"programs":[]`, which a merging fetch of the build modules answers since issue 1577 (mesh-sdk buildstep.MergeInto's Programs), added by hand to the captured bytes: a fixture without it is an older build module's, and its line warns that no compiled program is refused.
|
||||
|
||||
forge-get-file-go-mod.json is the answer of the forge's gitea_get_file (mesh-catalog modules/gitea, tools/index.ts) for modules/mesh-delivery-builder/go.mod of
|
||||
novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_compare for novox/mesh-sdk base main, head cb54756c3865 (a commit on the
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
|
||||
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"programs":[],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
|
||||
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"programs":[],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
|
||||
@@ -24,11 +26,17 @@ import (
|
||||
const (
|
||||
TryTool = "checker_try"
|
||||
TryReadTool = "checker_try_read"
|
||||
TryEndTool = "checker_try_end"
|
||||
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
||||
TriesState = "tries"
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
|
||||
MaxTries = 4
|
||||
// MaxTries and MaxTriesWaiting are checked when a try is asked, mesh-wide: a new try is refused while MaxTries tries run
|
||||
// on a build node or MaxTriesWaiting wait for one (queued, or their asks not yet taken by a node). It is no bound on
|
||||
// what runs later: tries let in while waiting may later all run at once, and each node's build capacity bounds what
|
||||
// runs on it. A try waiting behind the seat's other asks holds no node, so it no longer blocks the next try (issue
|
||||
// 1890: at 19:53 on 2026-10-26 four tries sat queued behind 44 checks for up to 47 minutes and no fifth could be asked).
|
||||
MaxTries = 4
|
||||
MaxTriesWaiting = 8
|
||||
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
|
||||
TryScan = 64
|
||||
// TryKeep is how long an ended try's key is kept before the checker removes it.
|
||||
@@ -75,9 +83,104 @@ type Try struct {
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||
Asks []string `json:"asks,omitempty"`
|
||||
// Steps say where each ask stands while the try runs and after: asked until a node takes it, then the node, its
|
||||
// state, the phase its step says and when (issue 1890: a 31-minute try could not be told queued from running).
|
||||
Steps []TryStep `json:"steps,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
|
||||
const TryAsked = "asked"
|
||||
|
||||
// TryStep is one ask of a try as its step key last said it.
|
||||
type TryStep struct {
|
||||
Ask string `json:"ask"`
|
||||
Node string `json:"node,omitempty"`
|
||||
State string `json:"state"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
PhaseAt time.Time `json:"phase-at,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// tryWriter writes a running try's record again as its asks are put and their steps move: one writer per try, holding
|
||||
// the revision it wrote last, so the try's end is written at that revision. A write refused (the try ended by hand)
|
||||
// stops the writes; the end's own write then says it.
|
||||
type tryWriter struct {
|
||||
mu sync.Mutex
|
||||
tr *Tryer
|
||||
ctx context.Context
|
||||
key string
|
||||
t Try
|
||||
rev uint64
|
||||
stopped bool
|
||||
}
|
||||
|
||||
// heard takes an ask put (step nil) or a value of its step key in, and writes the record when it changed.
|
||||
func (w *tryWriter) heard(askID string, step *delivery.Step) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return
|
||||
}
|
||||
changed := false
|
||||
if !slices.Contains(w.t.Asks, askID) {
|
||||
w.t.Asks, changed = append(w.t.Asks, askID), true
|
||||
}
|
||||
i := slices.IndexFunc(w.t.Steps, func(s TryStep) bool { return s.Ask == askID })
|
||||
if i < 0 {
|
||||
w.t.Steps, changed = append(w.t.Steps, TryStep{Ask: askID, State: TryAsked}), true
|
||||
i = len(w.t.Steps) - 1
|
||||
}
|
||||
if step != nil {
|
||||
now := TryStep{Ask: askID, Node: step.Node, State: step.State, Phase: step.Phase, Started: step.Started,
|
||||
PhaseAt: step.PhaseAt, Ended: step.Ended}
|
||||
changed = changed || now != w.t.Steps[i]
|
||||
w.t.Steps[i] = now
|
||||
}
|
||||
if changed {
|
||||
w.write()
|
||||
}
|
||||
}
|
||||
|
||||
// write writes the record at the revision written last; a refusal stops the writes.
|
||||
func (w *tryWriter) write() {
|
||||
data, err := encodeTry(w.t)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rev, err := w.tr.Tries.Update(w.ctx, w.key, data, w.rev)
|
||||
if err != nil {
|
||||
w.stopped = true
|
||||
w.tr.logf("%s: its asks' progress is no longer written: %v", w.key, err)
|
||||
return
|
||||
}
|
||||
w.rev = rev
|
||||
}
|
||||
|
||||
// end stops the writes and answers the record and its revision, for the try's end.
|
||||
func (w *tryWriter) end() (Try, uint64) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.stopped = true
|
||||
return w.t, w.rev
|
||||
}
|
||||
|
||||
// HoldsANode says whether the try holds a build node now: it runs, and a step of its asks runs on a node, or it names
|
||||
// no step (a checker that writes its asks only at the end, or one about to put its first).
|
||||
func (t Try) HoldsANode() bool {
|
||||
if t.State != TryRunning {
|
||||
return false
|
||||
}
|
||||
if len(t.Steps) == 0 {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(t.Steps, func(s TryStep) bool { return s.State == delivery.StepRunning })
|
||||
}
|
||||
|
||||
// Over says whether the try came to its end.
|
||||
@@ -119,8 +222,8 @@ func NotChecked(repository string) error {
|
||||
}
|
||||
|
||||
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
|
||||
// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a
|
||||
// repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
|
||||
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
|
||||
head := strings.ToLower(strings.TrimSpace(a.Head))
|
||||
@@ -183,7 +286,7 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var open []string
|
||||
var holding, waiting []string
|
||||
for i, cur := range tries {
|
||||
if !live[i] {
|
||||
continue
|
||||
@@ -191,10 +294,18 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
return cur.ID, nil
|
||||
}
|
||||
open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State))
|
||||
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
|
||||
if cur.HoldsANode() {
|
||||
holding = append(holding, said)
|
||||
} else {
|
||||
waiting = append(waiting, said)
|
||||
}
|
||||
}
|
||||
if len(open) >= MaxTries {
|
||||
return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", "))
|
||||
// Checked at this ask only: a try queued behind the seat's checks holds no node; tries let in while waiting may later
|
||||
// all run at once, and each node's build capacity bounds what runs on it.
|
||||
if len(holding) >= MaxTries || len(waiting) >= MaxTriesWaiting {
|
||||
return "", fmt.Errorf("a new try is refused while %d tries run on a build node or %d wait for one; now %d run and %d wait: running %s; waiting %s; ask again when one ends",
|
||||
MaxTries, MaxTriesWaiting, len(holding), len(waiting), orNone(holding), orNone(waiting))
|
||||
}
|
||||
t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10)
|
||||
data, err := encodeTry(t)
|
||||
@@ -222,6 +333,52 @@ func ReadTry(store TryStore, id string) (json.RawMessage, error) {
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// TryEnder is the tries state as the end tool sees it: one key with its revision, and a write at that revision.
|
||||
type TryEnder interface {
|
||||
Entry(key string) (json.RawMessage, uint64, error)
|
||||
Update(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
// EndTry ends a try that waits or runs as error, saying why and that it was ended by hand (issue 734): its place is
|
||||
// free at once, and the next checker_try of its head asks anew. A try that ended already is refused, naming its end.
|
||||
func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
|
||||
if !tryKeyOK.MatchString(id) {
|
||||
return Try{}, fmt.Errorf("%q is not a try's id, try-<digits>", id)
|
||||
}
|
||||
why = strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
return Try{}, errors.New("say why the try is ended: the why is kept in what it says")
|
||||
}
|
||||
raw, rev, err := store.Entry(id)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Try{}, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if t.Over() {
|
||||
return Try{}, fmt.Errorf("%s ended already, %s at %s", id, t.State, t.Ended.Format(time.RFC3339))
|
||||
}
|
||||
t.State, t.Ended = TryError, now.UTC()
|
||||
if !t.Started.IsZero() {
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
}
|
||||
t.Said = strings.TrimSpace("ended by hand (" + TryEndTool + "): " + why + " | " + t.Said)
|
||||
t.Said = strings.TrimSuffix(t.Said, " |")
|
||||
data, err := encodeTry(t)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if err := store.Update(id, data, rev); err != nil {
|
||||
return Try{}, fmt.Errorf("%s changed while it was ended, read it and end it again: %w", id, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// TryOptions narrow what a try runs; a pull request's check has none.
|
||||
type TryOptions struct {
|
||||
Level string
|
||||
@@ -234,12 +391,14 @@ type Tryer struct {
|
||||
Tries KV
|
||||
Judge Judge
|
||||
Bound time.Duration
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// NotTaken ends a try whose ask no build node took within it as error (issue 734); none when 0.
|
||||
NotTaken time.Duration
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
inflight map[string]bool
|
||||
inflight map[string]context.CancelFunc
|
||||
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
||||
}
|
||||
|
||||
@@ -267,25 +426,31 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
||||
tr.logf("tries.%s holds what is not a try, left: %v", key, err)
|
||||
return
|
||||
}
|
||||
if t.Over() {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
if tr.inflight == nil {
|
||||
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
|
||||
tr.inflight, tr.seen = map[string]context.CancelFunc{}, map[string]bool{}
|
||||
}
|
||||
if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
|
||||
if t.Over() {
|
||||
// Ended while it ran (checker_try_end): its judging stops, and its asks are no longer waited for.
|
||||
if stop := tr.inflight[key]; stop != nil {
|
||||
stop()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
tr.seen[key] = true
|
||||
tr.inflight[key] = true
|
||||
tr.inflight[key] = stop
|
||||
tr.mu.Unlock()
|
||||
tr.wg.Add(1)
|
||||
go func() {
|
||||
defer tr.wg.Done()
|
||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
|
||||
tr.run(ctx, key)
|
||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock(); stop() }()
|
||||
tr.run(run, key)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -336,27 +501,37 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
}
|
||||
return
|
||||
}
|
||||
cs := &Case{Key: key, Group: key,
|
||||
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
|
||||
cs := &Case{Key: key, Group: key, Watch: progress.heard,
|
||||
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
|
||||
Trunk: map[string]string{t.Repository: t.Trunk},
|
||||
Queued: t.Asked,
|
||||
Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
|
||||
if ctx.Err() != nil {
|
||||
return // left running: the checker takes it over when it starts
|
||||
judging := ctx
|
||||
if tr.NotTaken > 0 {
|
||||
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
|
||||
}
|
||||
out, judged := judgeWithin(judging, tr.Judge, cs, tr.Bound)
|
||||
if ctx.Err() != nil {
|
||||
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
|
||||
}
|
||||
t, rev = progress.end()
|
||||
v := out.Verdict
|
||||
t.Ended = tr.now().UTC()
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
t.Levels, t.Machines = v.Levels, v.Machines
|
||||
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||
for _, n := range cs.Notes() {
|
||||
t.Said += " | " + n
|
||||
}
|
||||
switch {
|
||||
case !judged || IsNodesFailure(v):
|
||||
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v):
|
||||
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
|
||||
t.State = TryError
|
||||
if IsLostAsk(v) || IsNotTaken(v) {
|
||||
t.Failed = []string{v.Failed.Words}
|
||||
}
|
||||
case v.State == delivery.VerdictPass:
|
||||
t.State = TryPass
|
||||
default:
|
||||
@@ -403,3 +578,11 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// orNone joins the tries said, or says none.
|
||||
func orNone(said []string) string {
|
||||
if len(said) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(said, ", ")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -23,6 +24,19 @@ func (m memTries) Create(key string, value []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// writtenTries is a memKV that keeps every running try it was given to write, in order.
|
||||
type writtenTries struct {
|
||||
*memKV
|
||||
running []Try
|
||||
}
|
||||
|
||||
func (w *writtenTries) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
|
||||
if t, err := DecodeTry(value); err == nil && !t.Over() {
|
||||
w.running = append(w.running, t)
|
||||
}
|
||||
return w.memKV.Update(ctx, key, value, rev)
|
||||
}
|
||||
|
||||
var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA}
|
||||
|
||||
func readTry(t testing.TB, kv *memKV, id string) Try {
|
||||
@@ -54,14 +68,38 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
|
||||
if err != nil || again != id {
|
||||
t.Errorf("a second ask of the same head answered %q, %v, want %q", again, err, id)
|
||||
}
|
||||
heads := []string{head2, trunkA2, trunkS2}
|
||||
for i, h := range heads {
|
||||
if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: h, Trunk: trunkA}, at.Add(time.Duration(i+2)*time.Second)); err != nil {
|
||||
t.Fatalf("try %d: %v", i+2, err)
|
||||
}
|
||||
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
|
||||
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
|
||||
onNode := func(i int) Try {
|
||||
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
|
||||
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
|
||||
}
|
||||
if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute)); err == nil || !strings.Contains(err.Error(), "4 tries wait or run") {
|
||||
t.Errorf("a fifth try answered %v, want refused naming the four", err)
|
||||
waitingFor := func(i int) Try {
|
||||
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
|
||||
Steps: []TryStep{{Ask: "a", State: TryAsked}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
holding, wait int
|
||||
refused string
|
||||
}{
|
||||
{"four queued behind the checks, none on a node: a fifth is taken", 0, 4, ""},
|
||||
{"four on a node", MaxTries, 0, "now 4 run and 0 wait"},
|
||||
{"three on a node, eight waiting", 3, MaxTriesWaiting, "a new try is refused while 4 tries run on a build node or 8 wait for one; now 3 run and 8 wait"},
|
||||
} {
|
||||
kv := newMemKV()
|
||||
for i := range c.holding + c.wait {
|
||||
tr := waitingFor(i + 1)
|
||||
if i < c.holding {
|
||||
tr = onNode(i + 1)
|
||||
}
|
||||
data, _ := encodeTry(tr)
|
||||
_, _ = kv.Create(context.Background(), tr.ID, data)
|
||||
}
|
||||
_, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute))
|
||||
if (c.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.refused)) {
|
||||
t.Errorf("%s: answered %v, want refused %q", c.what, err, c.refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +127,8 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
written := &writtenTries{memKV: kv}
|
||||
tr := &Tryer{Tries: written, Judge: b.ex}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
@@ -97,6 +136,21 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
|
||||
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
||||
t.Fatalf("try = %+v", got)
|
||||
}
|
||||
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
|
||||
// keeps each ask's node and state.
|
||||
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
|
||||
written.running[2].Steps[0].Node != "shanks" {
|
||||
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
|
||||
}
|
||||
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
|
||||
t.Errorf("the ended try's steps = %+v", got.Steps)
|
||||
}
|
||||
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
|
||||
if len(got.Asks) != 1 {
|
||||
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
|
||||
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
|
||||
}
|
||||
fetch := b.steps.tool("fetch")
|
||||
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them
|
||||
// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an
|
||||
// approved merge goes before a check not yet started. A bucket not there is read as empty.
|
||||
type Turns struct {
|
||||
mu sync.Mutex
|
||||
byRepo map[string]delivery.Turn
|
||||
}
|
||||
|
||||
// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that
|
||||
// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing.
|
||||
func (t *Turns) Put(key string, value []byte, deleted bool) bool {
|
||||
repository, err := delivery.ParseTurnsKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.byRepo == nil {
|
||||
t.byRepo = map[string]delivery.Turn{}
|
||||
}
|
||||
cur, had := t.byRepo[repository]
|
||||
if !deleted {
|
||||
if turn, err := delivery.DecodeTurn(value); err == nil {
|
||||
t.byRepo[repository] = turn
|
||||
return !had || !delivery.SameTurn(cur, turn)
|
||||
}
|
||||
}
|
||||
delete(t.byRepo, repository)
|
||||
return had
|
||||
}
|
||||
|
||||
// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that
|
||||
// waits first in it; false when none does.
|
||||
func (t *Turns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||
if t == nil {
|
||||
return "", delivery.TurnWait{}, false
|
||||
}
|
||||
sorted := make([]string, 0, len(repositories))
|
||||
for _, r := range repositories {
|
||||
sorted = append(sorted, strings.ToLower(r))
|
||||
}
|
||||
sort.Strings(sorted)
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for _, r := range sorted {
|
||||
turn, ok := t.byRepo[r]
|
||||
if !ok || !delivery.TurnHolds(turn, heads) {
|
||||
continue
|
||||
}
|
||||
for _, w := range turn.Waiting {
|
||||
if w.Heads != heads {
|
||||
return r, w, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", delivery.TurnWait{}, false
|
||||
}
|
||||
|
||||
// All is every turn held now, by repository (a copy).
|
||||
func (t *Turns) All() map[string]delivery.Turn {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
out := make(map[string]delivery.Turn, len(t.byRepo))
|
||||
for r, turn := range t.byRepo {
|
||||
out[r] = turn
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -93,11 +93,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
|
||||
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
@@ -124,8 +124,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||
// The try's two tools (ADR 0501 §5) beside them: mesh-controller's try calls them.
|
||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them.
|
||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "kept", "history": 1 },
|
||||
{ "name": "owed", "history": 5 },
|
||||
{ "name": "previews", "history": 5 },
|
||||
@@ -25,6 +26,7 @@
|
||||
"mesh-delivery-assigner.compositions",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-gate.turns",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
@@ -41,6 +43,7 @@
|
||||
"checker_pull",
|
||||
"checker_status",
|
||||
"checker_try",
|
||||
"checker_try_end",
|
||||
"checker_try_read",
|
||||
"checker_verdicts"
|
||||
],
|
||||
|
||||
@@ -91,6 +91,28 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
}
|
||||
}
|
||||
|
||||
// listedVia asks the forge for the files directly under one directory of a commit through call: gitea_commit_trees with
|
||||
// listed (issue 1126), answer {listed: {directory: {name: blob id}}}.
|
||||
func listedVia(ctx context.Context, call caller) bridge.Listed {
|
||||
return func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{dir}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Listed map[string]map[string]string `json:"listed"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {listed}: %w", commitTreesTool, err)
|
||||
}
|
||||
if a.Listed == nil {
|
||||
// A forge module from before listed answers none: placement cannot be read, never read as empty.
|
||||
return nil, fmt.Errorf("the answer of %s lists no directory; the forge module does not list yet", commitTreesTool)
|
||||
}
|
||||
return a.Listed[dir], nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
|
||||
@@ -117,8 +117,8 @@ type publisher interface {
|
||||
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files)
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, list bridge.Listed, logger *log.Logger) {
|
||||
msg, err := bridge.ConvertWith(m.Subject(), m.Data(), trees, files, list)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
@@ -237,7 +237,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger)
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), listedVia(hctx, call), logger)
|
||||
})
|
||||
},
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
})
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
@@ -97,7 +97,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -112,7 +112,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
|
||||
}
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0))
|
||||
if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 {
|
||||
t.Fatalf("calls %v, want the trees once and first", keys)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f h1:aWqn4HHTPUp/+28h7b8uFqQptAYvQFs05CALMHN7SkI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -77,6 +78,24 @@ type pull struct {
|
||||
ModuleDirsSaid bool `json:"module_dirs_said"`
|
||||
Base string `json:"base"`
|
||||
MergeSHA string `json:"merge_commit_sha"`
|
||||
// Paths are the files the pull request changes (a removed or renamed one included), PathsTruncated says the forge
|
||||
// cut the list; nil when the forge did not say them.
|
||||
Paths *[]string `json:"paths"`
|
||||
PathsTruncated bool `json:"paths_truncated"`
|
||||
}
|
||||
|
||||
// touchesPlacement says whether a pull request may change a placement file: a changed path under
|
||||
// delivery.PlacementDir, or paths the forge did not say or cut (the safe direction: the files are read).
|
||||
func (p pull) touchesPlacement() bool {
|
||||
if p.Paths == nil || p.PathsTruncated {
|
||||
return true
|
||||
}
|
||||
for _, path := range *p.Paths {
|
||||
if strings.HasPrefix(path, delivery.PlacementDir+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func unreadable(subject, why string, args ...any) error {
|
||||
@@ -88,6 +107,19 @@ func unreadable(subject, why string, args ...any) error {
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
|
||||
return ConvertWith(subject, data, trees, files, nil)
|
||||
}
|
||||
|
||||
// Listed asks the forge for the files directly under one directory of a commit, by name to blob id (gitea_commit_trees
|
||||
// with listed); a directory the commit lacks lists none.
|
||||
type Listed func(owner, repo, sha, dir string) (map[string]string, error)
|
||||
|
||||
// ConvertWith is Convert; with list, a merge of the catalogue repository (delivery.CatalogueRepository) also carries every
|
||||
// placement file of its merge commit (novox/hq ADR 0549 §2, issue 1126), read whole or not at all: a forge that cannot
|
||||
// say is ErrRetry, as for the manifests, so the indexer never takes a part of placement for all of it. So does an
|
||||
// updated pull request of the catalogue that touches placement/ (touchesPlacement), with every placement file at its
|
||||
// head, which the indexer writes as placement candidates (ADR 0549 §4: the nodes it places answer it before it merges).
|
||||
func ConvertWith(subject string, data []byte, trees Trees, files Files, list Listed) (Message, error) {
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
@@ -114,7 +146,7 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er
|
||||
return Message{}, unreadable(subject, "no branch")
|
||||
}
|
||||
if subject == MergedSubject && p.Base == Trunk {
|
||||
return merged(subject, p, trees, files)
|
||||
return merged(subject, p, trees, files, list)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
@@ -126,6 +158,13 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
|
||||
}
|
||||
}
|
||||
if action == delivery.PullUpdated && list != nil && delivery.SameRepository(f.Repository, delivery.CatalogueRepository) && p.touchesPlacement() {
|
||||
var err error
|
||||
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.HeadSHA, list, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the head %s: %v", ErrRetry, subject, f.Repository, p.HeadSHA, err)
|
||||
}
|
||||
f.PlacementsRead = true
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
@@ -227,7 +266,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma
|
||||
}
|
||||
|
||||
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
|
||||
func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
|
||||
func merged(subject string, p pull, trees Trees, files Files, list Listed) (Message, error) {
|
||||
if !sha40.MatchString(p.MergeSHA) {
|
||||
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
|
||||
}
|
||||
@@ -252,8 +291,15 @@ func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks})
|
||||
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
|
||||
if list != nil && delivery.SameRepository(repository, delivery.CatalogueRepository) {
|
||||
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.MergeSHA, list, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
f.PlacementsRead = true
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
@@ -350,3 +396,30 @@ func OpenMessage(answer []byte) (Message, error) {
|
||||
}
|
||||
return Message{Event: delivery.OpenEvent(), Data: body}, nil
|
||||
}
|
||||
|
||||
// placementsOf is every placement file of the catalogue at a commit, by node, its bytes exact: each `<node>.json` directly
|
||||
// under delivery.PlacementDir. A file whose name is no node is left out and the rest read; the catalogue's check refuses
|
||||
// such a file before it merges (ADR 0549 §1). Nil when there is none.
|
||||
func placementsOf(owner, repo, sha string, list Listed, files Files) (map[string]string, error) {
|
||||
names, err := list(owner, repo, sha, delivery.PlacementDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]string
|
||||
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||
file := delivery.PlacementDir + "/" + name
|
||||
node, ok := delivery.PlacementFileNode(file)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
raw, err := files(owner, repo, file, sha)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[node] = string(raw)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -79,6 +79,47 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
|
||||
t.Errorf("locks = %v, want %v", f.Locks, want)
|
||||
}
|
||||
if f.Placements != nil || f.PlacementsRead {
|
||||
t.Errorf("a merge of another repository than the catalogue carries placement: %v", f.Placements)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §2): a merge of the catalogue carries every placement file of its merge commit, by node; a file
|
||||
// whose name is no node is left out; a listing or a file the forge cannot give is a retry, never a part.
|
||||
catalogue := mergedBody(map[string]any{"repo": "mesh-catalog"})
|
||||
forge.have["placement/ace.json"] = []byte(`{"v":1,"node":"ace","modules":[]}`)
|
||||
forge.have["placement/g14.json"] = []byte(`{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`)
|
||||
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3", "a.b.json": "b4"}
|
||||
var listedAt []string
|
||||
list := func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
|
||||
return listing, nil
|
||||
}
|
||||
msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`}
|
||||
if !f.PlacementsRead || !reflect.DeepEqual(f.Placements, want) || !reflect.DeepEqual(listedAt, []string{"novox/mesh-catalog@" + merge + ":placement"}) {
|
||||
t.Errorf("placements = %v (read %v), listed at %v", f.Placements, f.PlacementsRead, listedAt)
|
||||
}
|
||||
listing = map[string]string{}
|
||||
if msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ = delivery.DecodeForge(msg.Data); !f.PlacementsRead || f.Placements != nil {
|
||||
t.Errorf("a catalogue with no placement file says it read none: %v, %v", f.Placements, f.PlacementsRead)
|
||||
}
|
||||
listing = map[string]string{"gone.json": "b5"}
|
||||
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
|
||||
t.Errorf("a placement file the forge cannot give: %v", err)
|
||||
}
|
||||
failing := func(_, _, _, _ string) (map[string]string, error) { return nil, errors.New("forge down") }
|
||||
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, failing); !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("a listing the forge cannot give: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
@@ -105,6 +146,55 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
|
||||
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §4): an updated pull request of the catalogue that touches placement/ carries every placement
|
||||
// file at its head, read whole or not at all; one that touches none, or of another repository, carries none and lists
|
||||
// nothing; paths the forge did not say or cut are read as touching; a closed one carries none.
|
||||
forge = &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest,
|
||||
"placement/ace.json": []byte(`{"v":1,"node":"ace","modules":[]}`), "placement/g14.json": []byte(`{"v":1,"node":"g14","modules":[]}`)}}
|
||||
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3"}
|
||||
var listedAt []string
|
||||
list := func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
|
||||
return listing, nil
|
||||
}
|
||||
all := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[]}`}
|
||||
catalogue := map[string]any{"repo": "mesh-catalog", "paths": []string{"placement/g14.json"}}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
subject string
|
||||
body map[string]any
|
||||
want map[string]string
|
||||
read bool
|
||||
}{
|
||||
{"a catalogue pull touching placement", updatedSubject, catalogue, all, true},
|
||||
{"a catalogue pull touching no placement", updatedSubject, map[string]any{"repo": "mesh-catalog"}, nil, false},
|
||||
{"a catalogue pull whose paths are cut", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths_truncated": true}, all, true},
|
||||
{"a catalogue pull whose paths are not said", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths": nil}, all, true},
|
||||
{"another repository's pull touching placement/", updatedSubject, map[string]any{"paths": []string{"placement/g14.json"}}, nil, false},
|
||||
{"a closed catalogue pull", closedSubject, catalogue, nil, false},
|
||||
} {
|
||||
listedAt = nil
|
||||
body := updated(row.body)
|
||||
msg, err := ConvertWith(row.subject, body, nil, forge.get, list)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if f.PlacementsRead != row.read || !reflect.DeepEqual(f.Placements, row.want) || (len(listedAt) > 0) != row.read {
|
||||
t.Errorf("%s: placements %v (read %v), listed %v", row.name, f.Placements, f.PlacementsRead, listedAt)
|
||||
}
|
||||
if row.read && listedAt[0] != "novox/mesh-catalog@"+head+":placement" {
|
||||
t.Errorf("%s: listed at %v, want the head", row.name, listedAt)
|
||||
}
|
||||
}
|
||||
listing = map[string]string{"gone.json": "b5"}
|
||||
if _, err := ConvertWith(updatedSubject, updated(catalogue), nil, forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
|
||||
t.Errorf("a placement file the forge cannot give at the head: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
|
||||
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
|
||||
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// asker writes a takeins value and waits for the indexer's answer to it.
|
||||
type asker func(key string, value []byte) (index.TakeInAsk, error)
|
||||
|
||||
// Tools are the indexer's tools.
|
||||
func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: index.TakeInTool,
|
||||
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
|
||||
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
|
||||
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
|
||||
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
|
||||
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
|
||||
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
|
||||
"indexer_module (mesh-delivery-view) reads the result.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.PlacementRereadTool,
|
||||
Description: "Read the catalogue's placement files again now (novox/hq ADR 0559 §8): the indexer asks the forge for the trunk's head and " +
|
||||
"every open catalogue pull request's, and writes its states placement (placement.<node>) and placement-candidates (<group>.<node>) " +
|
||||
"as a merge and a pull request's head would, only where a key differs; a candidate key whose file no longer differs, or whose pull " +
|
||||
"request is closed, is deleted. The recovery for a forge message the indexer missed. Answers the heads read, the keys written or " +
|
||||
"deleted in each state, or why nothing was (the forge could not say). indexer_placement (mesh-delivery-view) reads the result.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why it is read again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewPlacementReread(str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Reread, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// askTakeIn writes the takeins value at key and waits, on a watch of the key, at most bound for the indexer's answer.
|
||||
func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
|
||||
return func(key string, value []byte) (index.TakeInAsk, error) {
|
||||
answered := make(chan index.TakeInAsk, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
take := func(revision uint64, v []byte) bool {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if mine == 0 || revision <= mine {
|
||||
return false
|
||||
}
|
||||
a, err := index.DecodeTakeIn(v)
|
||||
if err != nil || !a.Done() {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case answered <- a:
|
||||
default:
|
||||
}
|
||||
return true
|
||||
}
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
if c.Op == "put" {
|
||||
take(c.Revision, c.Value)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return index.TakeInAsk{}, fmt.Errorf("watching %s.%s: %w", index.TakeInsState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil {
|
||||
take(e.Revision, e.Value)
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case a := <-answered:
|
||||
return a, nil
|
||||
case <-wait.Done():
|
||||
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
|
||||
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
||||
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
|
||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its
|
||||
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
|
||||
//
|
||||
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
|
||||
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
|
||||
@@ -90,6 +91,58 @@ func pinned(want string) *tls.Config {
|
||||
}
|
||||
}
|
||||
|
||||
// awaitedStore is a state bound when it appears (process.AwaitBucket) as the indexer's store: until then every call is
|
||||
// process.ErrNotYet.
|
||||
type awaitedStore struct{ b *process.AwaitedBucket }
|
||||
|
||||
func (s awaitedStore) store() (kvStore, error) {
|
||||
kv := s.b.KV()
|
||||
if kv == nil {
|
||||
return kvStore{}, process.ErrNotYet
|
||||
}
|
||||
return kvStore{kv}, nil
|
||||
}
|
||||
|
||||
func (s awaitedStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return st.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.Delete(ctx, key, revision)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Keys(ctx context.Context) ([]string, error) {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return st.Keys(ctx)
|
||||
}
|
||||
|
||||
// kvStore is the candidates bucket as the indexer's store: compare-and-set through Create, Update and Delete at a revision.
|
||||
type kvStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
@@ -139,6 +192,7 @@ type delivered interface {
|
||||
Subject() string
|
||||
Data() []byte
|
||||
Ack() error
|
||||
DoubleAck(ctx context.Context) error
|
||||
Term() error
|
||||
}
|
||||
|
||||
@@ -159,7 +213,12 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, logger *log.Log
|
||||
case err != nil:
|
||||
logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err)
|
||||
default:
|
||||
_ = m.Ack()
|
||||
// Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that
|
||||
// follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which
|
||||
// kicks the fill again.
|
||||
if err := m.DoubleAck(ctx); err != nil {
|
||||
logger.Printf("%s: %s#%d applied, its acknowledgement was not confirmed: %v", m.Subject(), f.Repository, f.Number, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,6 +311,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then
|
||||
// a merge's taken key is said and not written, and the rest of the indexer runs.
|
||||
taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf)
|
||||
// Its placement state (ADR 0549 §2) is bound when it appears, as every state a delivery adds (issue 1790): until then a
|
||||
// catalogue merge's placement is said and not written, and the rest of the indexer runs.
|
||||
placement := process.AwaitBucket(ctx, js, conn, delivery.PlacementBucket, "no catalogue merge's placement is written", logger.Printf)
|
||||
// Its placement candidates (issue 1126, ADR 0549 §4) are bound the same way: until the state appears a catalogue pull
|
||||
// request's placement is said and not written, so no node answers it and its check names every placed node as silent.
|
||||
placementCandidates := process.AwaitBucket(ctx, js, conn, delivery.PlacementCandidatesBucket, "no catalogue pull request's placement candidate is written", logger.Printf)
|
||||
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
|
||||
// such a module waits for its path, every other is indexed (issue 1945).
|
||||
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
|
||||
@@ -266,11 +334,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
errs := make(chan error, 1)
|
||||
var buildsWatch jetstream.KeyWatcher
|
||||
if builds != nil {
|
||||
if w, err := settle(ctx, builds, each); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
buildsWatch = w
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,8 +353,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
Placement: awaitedStore{placement}, PlacementCandidates: awaitedStore{placementCandidates}, ReadPlacement: placementVia(calls.Call), Say: func(event string, body any) error { return calls.Emit(event, body) }}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
@@ -336,6 +406,40 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||
}
|
||||
// A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records
|
||||
// name it and the bucket holds none (registered.go); the records that follow are heard below.
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
logger.Printf("indexing the registered modules at start: %v", err)
|
||||
}
|
||||
if buildsWatch != nil {
|
||||
go func() {
|
||||
errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) {
|
||||
each(e)
|
||||
if !opOf(e) {
|
||||
return
|
||||
}
|
||||
var r delivery.BuildRecord
|
||||
if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" {
|
||||
return
|
||||
}
|
||||
if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil {
|
||||
logger.Printf("the registered module %s is not indexed: %v", r.Module, err)
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
|
||||
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
|
||||
// merged message (index.RetryUnread).
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, index.TakeInsBucket, "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
|
||||
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
|
||||
process.NotWatched(logger.Printf, index.TakeInsBucket, err, "indexer_take_in is not answered until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
|
||||
}
|
||||
})
|
||||
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
|
||||
if err := ix.Reconcile(ctx); err != nil {
|
||||
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
|
||||
@@ -373,7 +477,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Essential: the forge's merges are what the indexer indexes.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
|
||||
// A merge applied while the taken state was missing, and the gate's heads once the forge stream is applied whole, get
|
||||
// their taken key (issue 1280): when the state appears (Ready runs once, now or then), and after each applied message.
|
||||
ix.Drained = func(ctx context.Context) (bool, error) {
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return info.NumPending == 0 && info.NumAckPending == 0, nil
|
||||
}
|
||||
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger); ix.KickFill(ctx) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -16,12 +16,18 @@ type fakeMsg struct {
|
||||
subject string
|
||||
data []byte
|
||||
acked, termd int
|
||||
doubled int // acked by DoubleAck, which waits for the server
|
||||
}
|
||||
|
||||
func (f *fakeMsg) Subject() string { return f.subject }
|
||||
func (f *fakeMsg) Data() []byte { return f.data }
|
||||
func (f *fakeMsg) Ack() error { f.acked++; return nil }
|
||||
func (f *fakeMsg) Term() error { f.termd++; return nil }
|
||||
func (f *fakeMsg) DoubleAck(context.Context) error {
|
||||
f.acked++
|
||||
f.doubled++
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMsg) Term() error { f.termd++; return nil }
|
||||
|
||||
type memStore struct{ vals map[string][]byte }
|
||||
|
||||
@@ -53,7 +59,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
|
||||
handle(context.Background(), m, ix, log.New(&bytes.Buffer{}, "", 0))
|
||||
c, err := delivery.DecodeCandidate(st.vals["feat/x"])
|
||||
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.termd != 0 {
|
||||
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 {
|
||||
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
|
||||
@@ -93,3 +98,182 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
|
||||
return *a.Contains, nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea gitea_get_file): arguments owner,
|
||||
// repo, path and ref, answer {file: {content}}; a path it does not have fails with a 404 in its words.
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles).
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// readVia reads a repository at a commit as the forge bridge does for a merged message: the trees by gitea_commit_trees,
|
||||
// then the module.json and lock files of the one directory dir by gitea_get_file (a repository can hold hundreds). A manifest the
|
||||
// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out.
|
||||
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||
if !ok {
|
||||
return index.RepoRead{}, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return index.RepoRead{}, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return index.RepoRead{}, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||
}
|
||||
out := index.RepoRead{Root: a.Root, Trees: a.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}}
|
||||
file := func(dir, name string) (string, error) {
|
||||
p := name
|
||||
if dir != "" {
|
||||
p = dir + "/" + name
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var f struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
|
||||
return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return *f.File.Content, nil
|
||||
}
|
||||
if _, held := a.Trees[dir]; held {
|
||||
m, err := file(dir, "module.json")
|
||||
if err != nil {
|
||||
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err)
|
||||
}
|
||||
out.Manifests[dir] = m
|
||||
for _, name := range lockFiles {
|
||||
c, err := file(dir, name)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), ": 404 ") {
|
||||
continue
|
||||
}
|
||||
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, name), err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(c))
|
||||
if out.Locks[dir] == nil {
|
||||
out.Locks[dir] = map[string]string{}
|
||||
}
|
||||
out.Locks[dir][name] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// The forge's tools the placement reread asks (indexer_placement_reread, novox/hq ADR 0559 §8): a repository's branches
|
||||
// with their commits, and every open pull request with its branch and head.
|
||||
const (
|
||||
listBranchesTool = "gitea.gitea_list_branches"
|
||||
openPullsTool = "gitea.gitea_open_pulls"
|
||||
placementTrunk = "main"
|
||||
)
|
||||
|
||||
// placementVia reads the catalogue's placement now through call: the trunk's head (gitea_list_branches), every placement
|
||||
// file at it, and each open pull request of the catalogue into the trunk (gitea_open_pulls) with every placement file at
|
||||
// its head, each read whole or not at all (gitea_commit_trees with listed, gitea_get_file).
|
||||
func placementVia(call toolCaller) func(ctx context.Context) (index.PlacementRead, error) {
|
||||
owner, repo, _ := strings.Cut(delivery.CatalogueRepository, "/")
|
||||
filesAt := func(sha string) (map[string]string, error) {
|
||||
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{delivery.PlacementDir}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Listed map[string]map[string]string `json:"listed"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.Listed == nil {
|
||||
return nil, fmt.Errorf("the answer of %s lists no directory: %v", commitTreesTool, err)
|
||||
}
|
||||
var out map[string]string
|
||||
for name := range a.Listed[delivery.PlacementDir] {
|
||||
file := delivery.PlacementDir + "/" + name
|
||||
node, ok := delivery.PlacementFileNode(file)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": file, "ref": sha})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s at %s: %w", file, sha, err)
|
||||
}
|
||||
var f struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s for %s is not {file: {content}}: %v", getFileTool, file, err)
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[node] = *f.File.Content
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return func(context.Context) (index.PlacementRead, error) {
|
||||
raw, err := call(listBranchesTool, map[string]string{"owner": owner, "repo": repo})
|
||||
if err != nil {
|
||||
return index.PlacementRead{}, err
|
||||
}
|
||||
var b struct {
|
||||
Branches []struct {
|
||||
Name string `json:"name"`
|
||||
Commit string `json:"commit"`
|
||||
} `json:"branches"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &b); err != nil {
|
||||
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {branches}: %w", listBranchesTool, err)
|
||||
}
|
||||
var out index.PlacementRead
|
||||
for _, br := range b.Branches {
|
||||
if br.Name == placementTrunk {
|
||||
out.Trunk = br.Commit
|
||||
}
|
||||
}
|
||||
if out.Trunk == "" {
|
||||
return index.PlacementRead{}, fmt.Errorf("%s names no branch %s of %s", listBranchesTool, placementTrunk, delivery.CatalogueRepository)
|
||||
}
|
||||
if out.Files, err = filesAt(out.Trunk); err != nil {
|
||||
return index.PlacementRead{}, err
|
||||
}
|
||||
raw, err = call(openPullsTool, map[string]any{})
|
||||
if err != nil {
|
||||
return index.PlacementRead{}, err
|
||||
}
|
||||
var o struct {
|
||||
Pulls *[]struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Base string `json:"base"`
|
||||
Head string `json:"head"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
} `json:"pulls"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
|
||||
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
|
||||
}
|
||||
for _, p := range *o.Pulls {
|
||||
if !delivery.SameRepository(p.Owner+"/"+p.Repo, delivery.CatalogueRepository) || p.Base != placementTrunk || p.Number <= 0 || p.Head == "" || p.HeadSHA == "" {
|
||||
continue
|
||||
}
|
||||
files, err := filesAt(p.HeadSHA)
|
||||
if err != nil {
|
||||
return index.PlacementRead{}, fmt.Errorf("%s#%d: %w", delivery.CatalogueRepository, p.Number, err)
|
||||
}
|
||||
out.Pulls = append(out.Pulls, index.PullPlacement{Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Files: files})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -50,3 +51,33 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
t.Error("the trunk asked for paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
var files []string
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case commitTreesTool:
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t1","modules/other":"t2"}}`), nil
|
||||
case getFileTool:
|
||||
files = append(files, body.(map[string]string)["path"])
|
||||
switch body.(map[string]string)["path"] {
|
||||
case "module.json":
|
||||
return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil
|
||||
case "go.sum":
|
||||
return json.RawMessage(`{"file":{"content":"x"}}`), nil
|
||||
}
|
||||
return nil, errors.New("Gitea API /x: 404 not found")
|
||||
}
|
||||
return nil, errors.New("unexpected " + key)
|
||||
}
|
||||
got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
|
||||
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
|
||||
}
|
||||
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
|
||||
t.Fatalf("read = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// takerIn is the part of the indexer a takeins value asks through.
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
RereadPlacement(ctx context.Context) index.PlacementReread
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// answered already (this indexer's own write) or deleted is left.
|
||||
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
a, err := index.DecodeTakeIn(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a take-in ask, left: %v", index.TakeInsState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
if a.Placement {
|
||||
r := ix.RereadPlacement(ctx)
|
||||
a.Reread, a.Answered = &r, time.Now().UTC()
|
||||
logf("a placement reread asked by %s (%s): %s", a.By, a.Why, r.Answer)
|
||||
} else {
|
||||
r := ix.TakeIn(ctx, a.Module)
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||
}
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", index.TakeInsState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f h1:aWqn4HHTPUp/+28h7b8uFqQptAYvQFs05CALMHN7SkI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,6 +2,8 @@ package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -88,3 +90,102 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
|
||||
}
|
||||
return pathOf(strings.Trim(best.Path, "/")), true
|
||||
}
|
||||
|
||||
// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record
|
||||
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
|
||||
// built at.
|
||||
type Registration struct {
|
||||
Module, Repository, Path, Commit, Ref string
|
||||
}
|
||||
|
||||
// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it
|
||||
// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a
|
||||
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
|
||||
// built from nox-mesh, and every record of them names that ref).
|
||||
func registered(r delivery.BuildRecord) bool {
|
||||
return r.State == "" && r.Failed == ""
|
||||
}
|
||||
|
||||
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
|
||||
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
|
||||
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
|
||||
// first that holds the module's manifest is the one indexed.
|
||||
func (b *Builds) RegistrationsOf(module string) []Registration {
|
||||
type place struct{ repository, path string }
|
||||
byPlace := map[place][]delivery.BuildRecord{}
|
||||
b.mu.Lock()
|
||||
for _, r := range b.records {
|
||||
if r.Module == module && registered(r) {
|
||||
k := place{delivery.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
|
||||
byPlace[k] = append(byPlace[k], r)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
|
||||
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
|
||||
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
|
||||
best := map[place]delivery.BuildRecord{}
|
||||
for k, recs := range byPlace {
|
||||
count, latest := map[string]int{}, map[string]delivery.BuildRecord{}
|
||||
for _, r := range recs {
|
||||
count[r.Ref]++
|
||||
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
|
||||
latest[r.Ref] = r
|
||||
}
|
||||
}
|
||||
var pick string
|
||||
first := true
|
||||
for ref, rec := range latest {
|
||||
switch {
|
||||
case first:
|
||||
case trunkLike[ref] != trunkLike[pick]:
|
||||
if !trunkLike[ref] {
|
||||
continue
|
||||
}
|
||||
case count[ref] != count[pick]:
|
||||
if count[ref] < count[pick] {
|
||||
continue
|
||||
}
|
||||
case !newer(rec, latest[pick]):
|
||||
continue
|
||||
}
|
||||
pick, first = ref, false
|
||||
}
|
||||
best[k] = latest[pick]
|
||||
}
|
||||
recs := slices.Collect(maps.Values(best))
|
||||
slices.SortFunc(recs, func(x, y delivery.BuildRecord) int {
|
||||
switch {
|
||||
case trunkLike[x.Ref] != trunkLike[y.Ref]:
|
||||
if trunkLike[x.Ref] {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
case newer(x, y):
|
||||
return -1
|
||||
case newer(y, x):
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
|
||||
})
|
||||
out := make([]Registration, 0, len(recs))
|
||||
for _, r := range recs {
|
||||
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Registered is the registrations of every module a record names, by module name.
|
||||
func (b *Builds) Registered() []Registration {
|
||||
b.mu.Lock()
|
||||
names := map[string]bool{}
|
||||
for _, r := range b.records {
|
||||
names[r.Module] = true
|
||||
}
|
||||
b.mu.Unlock()
|
||||
var out []Registration
|
||||
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||
out = append(out, b.RegistrationsOf(name)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -46,22 +48,50 @@ type Indexer struct {
|
||||
Modules Store
|
||||
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
|
||||
Manifests Store
|
||||
Builds *Builds
|
||||
// Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none.
|
||||
Taken Store
|
||||
// Now is the clock of a taken value; nil is time.Now.
|
||||
Now func() time.Time
|
||||
Builds *Builds
|
||||
// Named asks the forge for the git object of each named path of the module directories given, at a commit
|
||||
// (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
|
||||
// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
|
||||
// has none.
|
||||
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
|
||||
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
|
||||
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
|
||||
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
|
||||
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
|
||||
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
|
||||
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
||||
Logf func(format string, args ...any)
|
||||
// Placement is the placement bucket (delivery.PlacementBucket, ADR 0549 §2); nil writes no placement.
|
||||
Placement Store
|
||||
// PlacementCandidates is the placement candidates bucket (delivery.PlacementCandidatesBucket, issue 1126); nil writes
|
||||
// no candidate (placement_candidates.go).
|
||||
PlacementCandidates Store
|
||||
// ReadPlacement asks the forge for the catalogue's placement now, at the trunk's head and each open pull request's
|
||||
// (indexer_placement_reread, ADR 0559 §8); nil rereads none.
|
||||
ReadPlacement func(ctx context.Context) (PlacementRead, error)
|
||||
// Say says one of the indexer's events (placement-changed); nil says none.
|
||||
Say func(event string, body any) error
|
||||
// Drained says whether the indexer has applied every message of the forge stream (the consumer has none pending and none
|
||||
// unacknowledged); FillTaken writes the source's heads only then. Nil fills none from the source (taken.go).
|
||||
Drained func(ctx context.Context) (bool, error)
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
@@ -85,6 +115,16 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
if err := ix.applyMerged(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.writeTaken(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
||||
}
|
||||
if err := ix.applyPlacement(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
var digests map[string]string
|
||||
var held map[string]delivery.Module
|
||||
@@ -140,6 +180,14 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
||||
for _, p := range f.Open {
|
||||
named[p] = true
|
||||
}
|
||||
if ix.PlacementCandidates != nil {
|
||||
// A pull request the forge no longer has open keeps no placement candidate (issue 1126).
|
||||
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return !named[delivery.OpenPull{Repository: c.Pull.Repository, Number: c.Pull.Number}]
|
||||
}, "its pull request is not open"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
keys, err := ix.Store.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the candidates: %w", err)
|
||||
|
||||
@@ -17,8 +17,9 @@ const (
|
||||
|
||||
// memStore is the candidates bucket's compare-and-set behaviour in memory: a revision per write, a conflict when the
|
||||
// revision read is no longer the key's. Writes counts the writes that went through; beforeWrite lets a test change the
|
||||
// key under the indexer.
|
||||
// key under the indexer; unbound, when set, is every read's error (a state not yet on the bus).
|
||||
type memStore struct {
|
||||
unbound error
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
next uint64
|
||||
@@ -29,6 +30,9 @@ type memStore struct {
|
||||
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (m *memStore) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if m.unbound != nil {
|
||||
return nil, 0, false, m.unbound
|
||||
}
|
||||
v, ok := m.vals[key]
|
||||
return v, m.revs[key], ok, nil
|
||||
}
|
||||
|
||||
@@ -43,6 +43,8 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri
|
||||
}
|
||||
|
||||
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
|
||||
// A merge's placement files are written as its manifests are (issue 1126): folded here to hold the count.
|
||||
placementIsWrittenAtAMergeOnlyWhereItChanged(t)
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
|
||||
@@ -25,6 +25,13 @@ import (
|
||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
return ix.applyManifests(ctx, f, true)
|
||||
}
|
||||
|
||||
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
|
||||
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||
// drops none.
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -84,7 +91,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
}
|
||||
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
|
||||
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
|
||||
if len(f.Manifests) > 0 && every {
|
||||
if drop && len(f.Manifests) > 0 && every {
|
||||
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
@@ -94,22 +95,92 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
// writes is the values written by then; commit is each module's commit after.
|
||||
writes int
|
||||
commit map[string]string
|
||||
// taken is the merge the repository's taken key names after then (issue 1280, ADR 0556 clause 1).
|
||||
taken string
|
||||
}{
|
||||
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
|
||||
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
|
||||
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error {
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||
{name: "a merge touching none of them is still taken in", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error {
|
||||
m := mergedMsg(repo, three)
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a merge while the taken state is not on the bus yet is still taken in", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error { // issue 1790: the state is bound when it appears, so the indexer runs before it does
|
||||
ix.Taken = &memStore{unbound: process.ErrNotYet}
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a merge heard while the taken state was missing is filled when the state appears", writes: 2, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error { // issue 1280: the key was missing, the merge was taken in without it
|
||||
on := ix.Taken.(*memStore)
|
||||
key, _ := delivery.RepositoryKey(repo)
|
||||
delete(on.vals, key)
|
||||
ix.Taken = &memStore{unbound: process.ErrNotYet}
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
if err := ix.Apply(context.Background(), m); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.Taken = on
|
||||
return ix.FillTaken(context.Background()) // no Drained: only the merge applied is written, not the source's word
|
||||
}},
|
||||
{name: "a source head whose merge was not applied is not a taken key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "",
|
||||
then: func(ix *Indexer) error { // the forge stream still holds the merge: the consumer is not drained
|
||||
delete(ix.Taken.(*memStore).vals, takenKey(repo))
|
||||
ix.Drained = func(context.Context) (bool, error) { return false, nil }
|
||||
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.FillTaken(context.Background())
|
||||
}},
|
||||
{name: "a source head of a drained forge stream fills a missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error {
|
||||
delete(ix.Taken.(*memStore).vals, takenKey(repo))
|
||||
ix.Drained = func(context.Context) (bool, error) { return true, nil }
|
||||
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.FillTaken(context.Background())
|
||||
}},
|
||||
{name: "filling from an older source head leaves a later key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil }
|
||||
m := mergedMsg(repo, three)
|
||||
m.Merge = next
|
||||
if err := ix.Apply(context.Background(), m); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.Drained = func(context.Context) (bool, error) { return true, nil }
|
||||
return ix.FillTaken(context.Background()) // the source still names merge
|
||||
}},
|
||||
{name: "an earlier merge heard again after a later one leaves taken", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == next && in == merge, nil }
|
||||
m := mergedMsg(repo, three)
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error {
|
||||
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t)
|
||||
taken := newMem()
|
||||
ix.Taken = taken
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -129,6 +200,13 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
|
||||
}
|
||||
}
|
||||
key, _ := delivery.RepositoryKey(repo)
|
||||
if _, held := taken.vals[key]; !held && tc.taken == "" {
|
||||
return
|
||||
}
|
||||
if got, err := delivery.DecodeTaken(taken.vals[key]); err != nil || got.Merge != tc.taken || got.Repository != repo {
|
||||
t.Errorf("taken.%s = %+v (%v), want merge %.8s", key, got, err, tc.taken)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -252,3 +330,8 @@ func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { //
|
||||
t.Errorf("module = %+v, want the held directory kept", m)
|
||||
}
|
||||
}
|
||||
|
||||
func takenKey(repo string) string {
|
||||
key, _ := delivery.RepositoryKey(repo)
|
||||
return key
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// Placement of modules on nodes (novox/hq ADR 0549 §2, issue 1126): at each merge of the catalogue the forge bridge
|
||||
// carries every placement file of the merge commit, and the indexer writes each node's key `placement.<node>` in its
|
||||
// state `placement` only when it differs (the revision alone is no change), deletes the key of a node whose file is
|
||||
// gone, and says `placement-changed` for each node it wrote or deleted, naming the modules added, removed or changed.
|
||||
// A merge that changes no placement writes nothing and says nothing.
|
||||
//
|
||||
// A file that does not read as a placement, or names another node than its file, is said in the log and its key is left
|
||||
// as it was: the catalogue's own check refuses such a file before it merges (ADR 0549 §1), so it is a fault of that
|
||||
// check, and a node keeps composing from the placement it had rather than from none.
|
||||
|
||||
// applyPlacement takes the placement files of a merged message in; a message that read none (another repository, an
|
||||
// older bridge) changes nothing.
|
||||
func (ix *Indexer) applyPlacement(ctx context.Context, f delivery.Forge) error {
|
||||
if ix.Placement == nil || f.Kind != delivery.ForgeMerged || !f.PlacementsRead {
|
||||
return nil
|
||||
}
|
||||
keys, err := ix.Placement.Keys(ctx)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
// The state is not on the bus yet: said, and the merge's candidates are still taken out (issue 1790's binding
|
||||
// writes the next catalogue merge's placement once it appears).
|
||||
ix.logf("%s at %s: its placement files are not written: %v", f.Repository, f.Merge, err)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the placement keys: %w", err)
|
||||
}
|
||||
gone := map[string]bool{}
|
||||
for _, key := range keys {
|
||||
if node, err := delivery.ParsePlacementKey(key); err == nil {
|
||||
gone[node] = true
|
||||
}
|
||||
}
|
||||
var refused []string
|
||||
for _, node := range slices.Sorted(maps.Keys(f.Placements)) {
|
||||
delete(gone, node)
|
||||
next, err := delivery.DecodePlacement([]byte(f.Placements[node]))
|
||||
if err == nil && next.Node != node {
|
||||
err = fmt.Errorf("the file of %s names the node %q", node, next.Node)
|
||||
}
|
||||
if err != nil {
|
||||
refused = append(refused, node)
|
||||
ix.logf("%s at %s: the placement file of %s is refused and placement.%s is left as it was (the catalogue's check should have refused it): %v",
|
||||
f.Repository, f.Merge, node, node, err)
|
||||
continue
|
||||
}
|
||||
next.Revision = f.Merge
|
||||
if err := ix.writePlacement(ctx, node, &next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(gone)) {
|
||||
if err := ix.writePlacement(ctx, node, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Said, and the merge's candidates are still taken out: a refused file holds nobody's delivery.
|
||||
ix.logf("%s at %s: the placement files of %v do not read; their keys are left as they were", f.Repository, f.Merge, refused)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writePlacement writes one node's key, or deletes it for next nil, only when it differs, and says what changed.
|
||||
func (ix *Indexer) writePlacement(ctx context.Context, node string, next *delivery.Placement) error {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var change *delivery.PlacementChange
|
||||
err = write(ctx, ix.Placement, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
change = nil
|
||||
var before delivery.Placement
|
||||
if found {
|
||||
cur, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
ix.logf("placement.%s holds what is not a placement, written again: %v", node, err)
|
||||
} else {
|
||||
before = cur
|
||||
}
|
||||
}
|
||||
if next == nil {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
change = &c
|
||||
return nil, true, nil
|
||||
}
|
||||
if found && delivery.SamePlacement(before, *next) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := delivery.EncodePlacement(*next)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c := delivery.ChangeOf(before, *next)
|
||||
change = &c
|
||||
return data, false, nil
|
||||
})
|
||||
if err != nil || change == nil {
|
||||
return err
|
||||
}
|
||||
if ix.Say != nil {
|
||||
if err := ix.Say(delivery.PlacementChangedEvent, *change); err != nil {
|
||||
// The key is written: a reader that watches it hears the change; the event is said loudly as not said.
|
||||
ix.logf("placement.%s is written, and %s could not be said: %v", node, delivery.PlacementChangedEvent, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// Placement candidates (issue 1126; novox/hq ADR 0549 §4: a placement change passes the composition check before any
|
||||
// node sees it). An updated pull request of the catalogue that touches placement/ carries every placement file at its
|
||||
// head (the forge bridge, PlacementsRead). For each node whose file at the head differs from the trunk's key
|
||||
// `placement.<node>` (SamePlacement, the revision left out), or whose file the head removes, the indexer writes the key
|
||||
// `<group>.<node>` of its state `placement-candidates`; a node whose file is the trunk's has none, and a key the head no
|
||||
// longer needs is deleted. The key is written only when its value changes, so its write is the node's event to compose
|
||||
// in shadow and answer. When the pull request closes or merges, or an open message no longer names it, its keys are
|
||||
// deleted, and so are they when a head the forge says the changes of (DirsKnown) carries no placement: it touches none.
|
||||
// A pull message that says nothing of its head (a resend of the open pull requests) leaves them as they are.
|
||||
//
|
||||
// A file at the head that does not read as a placement, or names another node, is said in the log and that node gets no
|
||||
// key (one it had is deleted): the catalogue's own check refuses the pull request (ADR 0549 §1), and a node answers no
|
||||
// candidate that is not one.
|
||||
|
||||
// applyPlacementCandidates takes one forge message into the placement candidates.
|
||||
func (ix *Indexer) applyPlacementCandidates(ctx context.Context, f delivery.Forge) error {
|
||||
if ix.PlacementCandidates == nil || !delivery.SameRepository(f.Repository, delivery.CatalogueRepository) || trunkLike[f.Branch] {
|
||||
return nil
|
||||
}
|
||||
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
|
||||
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
|
||||
}, fmt.Sprintf("%s#%d %s", f.Repository, f.Number, map[bool]string{true: "merged", false: "closed"}[f.Kind == delivery.ForgeMerged]))
|
||||
}
|
||||
if f.Kind != delivery.ForgePull {
|
||||
return nil
|
||||
}
|
||||
if !f.PlacementsRead {
|
||||
if f.DirsKnown {
|
||||
// The forge said what this head changes (its directories and paths) and the bridge read no placement: the head
|
||||
// changes no placement file, so a key an earlier head left goes (ADR 0559 §3).
|
||||
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
|
||||
}, fmt.Sprintf("%s#%d at %s changes no placement file", f.Repository, f.Number, f.Head))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
trunk, err := ix.trunkPlacements(ctx)
|
||||
if err != nil {
|
||||
ix.logf("%s#%d at %s: its placement candidates are not written: %v", f.Repository, f.Number, f.Head, err)
|
||||
return nil
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for node := range f.Placements {
|
||||
nodes[node] = true
|
||||
}
|
||||
for node := range trunk {
|
||||
nodes[node] = true
|
||||
}
|
||||
pull := delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}
|
||||
for _, node := range slices.Sorted(maps.Keys(nodes)) {
|
||||
key, err := delivery.PlacementCandidateKey(f.Branch, node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
}
|
||||
var next *delivery.PlacementCandidate // nil: no key for this node
|
||||
raw, atHead := f.Placements[node]
|
||||
before, inTrunk := trunk[node]
|
||||
switch {
|
||||
case atHead:
|
||||
p, err := delivery.DecodePlacement([]byte(raw))
|
||||
if err == nil && p.Node != node {
|
||||
err = fmt.Errorf("the file of %s names the node %q", node, p.Node)
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("%s#%d at %s: the placement file of %s is refused, so %s has no placement candidate (the catalogue's check refuses the pull request): %v",
|
||||
f.Repository, f.Number, f.Head, node, node, err)
|
||||
break
|
||||
}
|
||||
if inTrunk && delivery.SamePlacement(before, p) {
|
||||
break
|
||||
}
|
||||
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull, Placement: &p}
|
||||
case inTrunk:
|
||||
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull} // the head removes the node's file
|
||||
}
|
||||
if err := ix.writePlacementCandidate(ctx, key, next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// A node the head and the trunk both no longer name keeps no key of this group.
|
||||
return ix.dropPlacementCandidates(ctx, func(node string, c delivery.PlacementCandidate) bool {
|
||||
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number && !nodes[node]
|
||||
}, fmt.Sprintf("%s#%d at %s", f.Repository, f.Number, f.Head))
|
||||
}
|
||||
|
||||
// trunkPlacements is every node's placement in the trunk's state placement, by node.
|
||||
func (ix *Indexer) trunkPlacements(ctx context.Context) (map[string]delivery.Placement, error) {
|
||||
out := map[string]delivery.Placement{}
|
||||
if ix.Placement == nil {
|
||||
return nil, errors.New("the indexer reads no placement state")
|
||||
}
|
||||
keys, err := ix.Placement.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the placement keys: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
value, _, found, err := ix.Placement.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s holds what is not a placement: %w", key, err)
|
||||
}
|
||||
out[node] = p
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// writePlacementCandidate writes one candidate key, or deletes it for next nil, only when it changes.
|
||||
func (ix *Indexer) writePlacementCandidate(ctx context.Context, key string, next *delivery.PlacementCandidate) error {
|
||||
var data []byte
|
||||
if next != nil {
|
||||
var err error
|
||||
if data, err = delivery.EncodePlacementCandidate(*next); err != nil {
|
||||
return fmt.Errorf("%w: %s.%s: %v", ErrUnreadable, delivery.PlacementCandidatesState, key, err)
|
||||
}
|
||||
}
|
||||
err := write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
switch {
|
||||
case next == nil:
|
||||
return nil, found, nil
|
||||
case found && string(value) == string(data):
|
||||
return nil, false, nil
|
||||
}
|
||||
return data, false, nil
|
||||
})
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("%s.%s is not written: %v", delivery.PlacementCandidatesState, key, err)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// dropPlacementCandidates deletes every candidate key whose node and value gone says, and says each with why.
|
||||
func (ix *Indexer) dropPlacementCandidates(ctx context.Context, gone func(node string, c delivery.PlacementCandidate) bool, why string) error {
|
||||
keys, err := ix.PlacementCandidates.Keys(ctx)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the placement candidate keys: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
_, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dropped := false
|
||||
err = write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
ix.logf("%s.%s holds what is not a placement candidate, deleted: %v", delivery.PlacementCandidatesState, key, err)
|
||||
return nil, true, nil
|
||||
}
|
||||
dropped = gone(node, c)
|
||||
return nil, dropped, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dropped {
|
||||
ix.logf("%s.%s is deleted: %s", delivery.PlacementCandidatesState, key, why)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PlacementRead is the catalogue's placement as the forge has it now (indexer_placement_reread, ADR 0559 §8): the trunk's
|
||||
// head and every placement file at it, by node, and each open pull request into the trunk with its own.
|
||||
type PlacementRead struct {
|
||||
Trunk string
|
||||
Files map[string]string
|
||||
Pulls []PullPlacement
|
||||
}
|
||||
|
||||
// PullPlacement is one open catalogue pull request and every placement file at its head, by node.
|
||||
type PullPlacement struct {
|
||||
Branch string
|
||||
Number int
|
||||
Head string
|
||||
Files map[string]string
|
||||
}
|
||||
|
||||
// PlacementReread is the answer of indexer_placement_reread: the heads read, the keys written or deleted in each state,
|
||||
// and in words what happened.
|
||||
type PlacementReread struct {
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Pulls []string `json:"pulls,omitempty"`
|
||||
Placement []string `json:"placement"`
|
||||
Candidates []string `json:"candidates"`
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
|
||||
// RereadPlacement reads the catalogue's placement now, at the trunk's head and at the head of each open catalogue pull
|
||||
// request, and writes placement and placement-candidates as a merged and a pull message would (ADR 0549 §2, ADR 0559
|
||||
// §3), only where a key differs; a candidate key of a pull request that is no longer open, or whose file no longer
|
||||
// differs at its head, is deleted. It is the recovery for a forge message the indexer missed (ADR 0559 §8).
|
||||
func (ix *Indexer) RereadPlacement(ctx context.Context) PlacementReread {
|
||||
out := PlacementReread{Placement: []string{}, Candidates: []string{}}
|
||||
if ix.ReadPlacement == nil || ix.Placement == nil || ix.PlacementCandidates == nil {
|
||||
out.Answer = "the indexer reads no placement here: it has no forge reader or no placement state"
|
||||
return out
|
||||
}
|
||||
read, err := ix.ReadPlacement(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the forge could not say the catalogue's placement, nothing is written: " + err.Error()
|
||||
return out
|
||||
}
|
||||
out.Trunk = read.Trunk
|
||||
before := revisions(ctx, ix.Placement)
|
||||
beforeC := revisions(ctx, ix.PlacementCandidates)
|
||||
if err := ix.applyPlacement(ctx, delivery.Forge{Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Merge: read.Trunk,
|
||||
Placements: read.Files, PlacementsRead: true}); err != nil {
|
||||
out.Answer = "the trunk's placement could not be written: " + err.Error()
|
||||
return out
|
||||
}
|
||||
open := map[int]bool{}
|
||||
var failed []string
|
||||
for _, p := range read.Pulls {
|
||||
open[p.Number] = true
|
||||
out.Pulls = append(out.Pulls, fmt.Sprintf("%s#%d at %s", delivery.CatalogueRepository, p.Number, p.Head))
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: p.Branch, Number: p.Number, Head: p.Head,
|
||||
Action: delivery.PullUpdated, Placements: p.Files, PlacementsRead: true}
|
||||
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
|
||||
failed = append(failed, fmt.Sprintf("#%d: %v", p.Number, err))
|
||||
}
|
||||
}
|
||||
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return delivery.SameRepository(c.Pull.Repository, delivery.CatalogueRepository) && !open[c.Pull.Number]
|
||||
}, "its pull request is not open (indexer_placement_reread)"); err != nil {
|
||||
failed = append(failed, err.Error())
|
||||
}
|
||||
out.Placement = changed(before, revisions(ctx, ix.Placement))
|
||||
out.Candidates = changed(beforeC, revisions(ctx, ix.PlacementCandidates))
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
out.Answer = "read at the trunk's head " + read.Trunk + "; some candidates are not written: " + fmt.Sprint(failed)
|
||||
case len(out.Placement)+len(out.Candidates) == 0:
|
||||
out.Answer = "read at the trunk's head " + read.Trunk + " and every open catalogue pull request's: every key was as the forge has it, nothing written"
|
||||
default:
|
||||
out.Answer = fmt.Sprintf("read at the trunk's head %s and every open catalogue pull request's: %d placement keys and %d candidate keys written or deleted",
|
||||
read.Trunk, len(out.Placement), len(out.Candidates))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// revisions is every key of a store with its revision; a state not on the bus yet has none.
|
||||
func revisions(ctx context.Context, st Store) map[string]uint64 {
|
||||
out := map[string]uint64{}
|
||||
keys, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, key := range keys {
|
||||
if _, rev, found, err := st.Get(ctx, key); err == nil && found {
|
||||
out[key] = rev
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changed is every key written, created or deleted between two revisions maps, sorted.
|
||||
func changed(before, after map[string]uint64) []string {
|
||||
out := []string{}
|
||||
for key, rev := range after {
|
||||
if before[key] != rev {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
for key := range before {
|
||||
if _, ok := after[key]; !ok {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0549 §2 (issue 1126): a merge changing one node's file writes that key alone and says one event; a merge with no
|
||||
// placement change writes nothing and says nothing; a node whose file is gone loses its key and says its modules
|
||||
// removed; a file that does not read leaves its key; a message that read no placement changes none.
|
||||
func placementIsWrittenAtAMergeOnlyWhereItChanged(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _, logs := newIndexer(t)
|
||||
place := newMem()
|
||||
ix.Placement = place
|
||||
var said []delivery.PlacementChange
|
||||
ix.Say = func(event string, body any) error {
|
||||
if event != delivery.PlacementChangedEvent {
|
||||
t.Errorf("said %s", event)
|
||||
}
|
||||
said = append(said, body.(delivery.PlacementChange))
|
||||
return nil
|
||||
}
|
||||
merged := func(merge string, files map[string]string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Branch: "main",
|
||||
Number: 7, Head: "h", Merge: merge, Root: "r", Trees: map[string]string{"": "t"}, Placements: files, PlacementsRead: true}
|
||||
}
|
||||
ace := `{"v":1,"node":"ace","modules":[{"module":"plex"}]}`
|
||||
g14 := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
|
||||
written := func(node string) (delivery.Placement, uint64) {
|
||||
t.Helper()
|
||||
v, ok := place.vals["placement."+node]
|
||||
if !ok {
|
||||
return delivery.Placement{}, 0
|
||||
}
|
||||
p, err := delivery.DecodePlacement(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p, place.revs["placement."+node]
|
||||
}
|
||||
|
||||
if err := ix.Apply(ctx, merged("m1", map[string]string{"ace": ace, "g14": g14})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("ace"); p.Revision != "m1" || !reflect.DeepEqual(p.Assigned(), []string{"plex"}) || len(said) != 2 {
|
||||
t.Fatalf("first merge: %+v, said %v", p, said)
|
||||
}
|
||||
_, aceRev := written("ace")
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m2", map[string]string{"ace": ace, "g14": g14})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, rev := written("ace"); rev != aceRev || p.Revision != "m1" || len(said) != 0 {
|
||||
t.Errorf("a merge with no placement change wrote %+v (revision %d, was %d) and said %v", p, rev, aceRev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m3", map[string]string{"ace": ace, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"xterm"}]}`})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, rev := written("ace"); rev != aceRev || len(said) != 1 || said[0].Node != "g14" || said[0].Revision != "m3" ||
|
||||
!reflect.DeepEqual(said[0].Added, []string{"xterm"}) {
|
||||
t.Errorf("one file changed: ace's revision %d (was %d), said %+v", rev, aceRev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m4", map[string]string{"ace": ace, "g14": `{"v":1,"node":"ace","modules":[]}`})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("g14"); len(p.Modules) != 2 || len(said) != 0 || !strings.Contains(logs.String(), "the placement file of %s is refused") {
|
||||
t.Errorf("a file naming another node replaced the key: %+v, said %v, logs %s", p, said, logs)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m5", map[string]string{"ace": ace})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, rev := written("g14"); rev != 0 || len(said) != 1 || !reflect.DeepEqual(said[0].Removed, []string{"xterm", "zsh"}) {
|
||||
t.Errorf("a file removed: g14's key at revision %d, said %+v", rev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
unread := merged("m6", nil)
|
||||
unread.PlacementsRead = false
|
||||
if err := ix.Apply(ctx, unread); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("ace"); p.Revision != "m1" || len(said) != 0 {
|
||||
t.Errorf("a message that read no placement changed it: %+v, %v", p, said)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §4), placement candidates: the trunk now places ace alone. A catalogue pull request writes
|
||||
// `<group>.<node>` for each node whose file at its head differs from the trunk's, or that the head removes, and only
|
||||
// when the value changes; a pull message that read no placement, or a pull request changing none, writes nothing; a
|
||||
// file that does not read gets no key; a merge, a close or an open message not naming it deletes its keys.
|
||||
cands := newMem()
|
||||
ix.PlacementCandidates = cands
|
||||
said = nil
|
||||
pullOf := func(branch string, number int, head string, files map[string]string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: branch,
|
||||
Number: number, Head: head, Action: delivery.PullUpdated, Placements: files, PlacementsRead: files != nil}
|
||||
}
|
||||
candidate := func(key string) (delivery.PlacementCandidate, uint64, bool) {
|
||||
t.Helper()
|
||||
v, ok := cands.vals[key]
|
||||
if !ok {
|
||||
return delivery.PlacementCandidate{}, 0, false
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, cands.revs[key], true
|
||||
}
|
||||
keys := func() string { k, _ := cands.Keys(ctx); slices.Sort(k); return strings.Join(k, ",") }
|
||||
g14New := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
f delivery.Forge
|
||||
keys string
|
||||
}{
|
||||
{"a pull request adding g14", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
|
||||
{"the same head again", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
|
||||
{"a pull message that read no placement", pullOf("feat/p", 9, h2, nil), "feat/p.g14"},
|
||||
{"a pull request changing no placement", pullOf("feat/q", 10, h1, map[string]string{"ace": ace}), "feat/p.g14"},
|
||||
{"a new head removing ace", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"a head whose g14 file does not read", pullOf("feat/p", 9, h3, map[string]string{"g14": `{"v":1,"node":"ace","modules":[]}`}), "feat/p.ace"},
|
||||
{"the head adding g14 again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"a head the forge says changes no placement file", func() delivery.Forge { f := pullOf("feat/p", 9, h3, nil); f.DirsKnown = true; return f }(), ""},
|
||||
{"the head adding g14 and removing ace again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"an open message not naming it", delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: []delivery.OpenPull{{Repository: delivery.CatalogueRepository, Number: 10}}}, ""},
|
||||
{"the pull request heard again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"its merge", func() delivery.Forge {
|
||||
m := merged("m7", map[string]string{"g14": g14New})
|
||||
m.Branch, m.Number = "feat/p", 9
|
||||
return m
|
||||
}(), ""},
|
||||
} {
|
||||
_, revBefore, _ := candidate("feat/p.g14")
|
||||
if err := ix.Apply(ctx, row.f); err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if got := keys(); got != row.keys {
|
||||
t.Errorf("%s: keys %q, want %q", row.name, got, row.keys)
|
||||
}
|
||||
c, rev, ok := candidate("feat/p.g14")
|
||||
switch row.name {
|
||||
case "a pull request adding g14":
|
||||
if !ok || c.Group != "feat/p" || c.Pull != (delivery.Head{Repository: delivery.CatalogueRepository, Number: 9, Head: h1}) || c.Placement == nil ||
|
||||
!reflect.DeepEqual(c.Placement.Assigned(), []string{"zsh"}) {
|
||||
t.Errorf("%s: %+v", row.name, c)
|
||||
}
|
||||
case "the same head again", "a pull message that read no placement", "a pull request changing no placement":
|
||||
if rev != revBefore {
|
||||
t.Errorf("%s: feat/p.g14 written again (revision %d, was %d)", row.name, rev, revBefore)
|
||||
}
|
||||
case "a new head removing ace":
|
||||
if a, _, _ := candidate("feat/p.ace"); a.Placement != nil || a.Pull.Head != h2 || c.Pull.Head != h2 {
|
||||
t.Errorf("%s: ace %+v, g14 at %s", row.name, a, c.Pull.Head)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p, _ := written("g14"); len(p.Modules) != 1 || p.Revision != "m7" {
|
||||
t.Errorf("the merge's trunk placement of g14: %+v", p)
|
||||
}
|
||||
|
||||
// ADR 0559 §8, indexer_placement_reread: a reread with nothing changed writes nothing; one after a missed merge and a
|
||||
// missed head writes those keys; one after a missed revert deletes the candidate; a forge that cannot say writes nothing.
|
||||
g14Other := `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"htop"}]}`
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
read PlacementRead
|
||||
err error
|
||||
placement, cands []string
|
||||
}{
|
||||
{"nothing changed", PlacementRead{Trunk: "m7", Files: map[string]string{"g14": g14New}}, nil, []string{}, []string{}},
|
||||
{"a missed merge and a missed head", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
|
||||
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h1, Files: map[string]string{"g14": g14Other, "ace": ace}}}}, nil,
|
||||
[]string{"placement.ace"}, []string{"feat/r.g14"}},
|
||||
{"a missed revert", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
|
||||
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h2, Files: map[string]string{"g14": g14New, "ace": ace}}}}, nil,
|
||||
[]string{}, []string{"feat/r.g14"}},
|
||||
{"a forge that cannot say", PlacementRead{}, context.DeadlineExceeded, []string{}, []string{}},
|
||||
} {
|
||||
ix.ReadPlacement = func(context.Context) (PlacementRead, error) { return row.read, row.err }
|
||||
r := ix.RereadPlacement(ctx)
|
||||
if !reflect.DeepEqual(r.Placement, row.placement) || !reflect.DeepEqual(r.Candidates, row.cands) || r.Answer == "" {
|
||||
t.Errorf("%s: %+v", row.name, r)
|
||||
}
|
||||
}
|
||||
if keys() != "" {
|
||||
t.Errorf("after the reread of a revert, candidates %q", keys())
|
||||
}
|
||||
}
|
||||
@@ -164,6 +164,7 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
|
||||
}
|
||||
|
||||
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
|
||||
t.Run("the same preview again", testTheSamePreviewAppliedAgainWritesNothing)
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -176,7 +177,7 @@ func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
|
||||
func testTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
|
||||
// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge
|
||||
// followed the registration. The controller's outcome records name each registered module's repository, directory and
|
||||
// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the
|
||||
// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged
|
||||
// message as for every other repository.
|
||||
|
||||
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
|
||||
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
|
||||
type RepoRead struct {
|
||||
Root string
|
||||
Trees map[string]string
|
||||
Manifests map[string]string
|
||||
Locks map[string]map[string]string
|
||||
}
|
||||
|
||||
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
|
||||
// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The
|
||||
// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or
|
||||
// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could
|
||||
// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision
|
||||
// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945).
|
||||
func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error {
|
||||
_, err := ix.indexRegistered(ctx, r, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not.
|
||||
type taken struct {
|
||||
stored, held bool
|
||||
repository, commit, dir string
|
||||
why string
|
||||
}
|
||||
|
||||
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
|
||||
if ix.Modules == nil || ix.Read == nil {
|
||||
return taken{why: "the indexer reads no forge or writes no modules"}, nil
|
||||
}
|
||||
key, err := delivery.ModulesKey(r.Module)
|
||||
if err != nil {
|
||||
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
|
||||
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if found {
|
||||
ix.markRead(r.Module)
|
||||
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
|
||||
}
|
||||
repository := delivery.RepositoryName(r.Repository)
|
||||
commit := r.Commit
|
||||
dir := strings.Trim(r.Path, "/")
|
||||
if dir == delivery.RootPath {
|
||||
dir = ""
|
||||
}
|
||||
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
|
||||
// merge removed is absent at the head and is said and left.
|
||||
ix.mu.Lock()
|
||||
for repo, head := range ix.source.heads {
|
||||
if delivery.SameRepository(repo, r.Repository) && head != "" {
|
||||
commit = head
|
||||
}
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
t := taken{repository: repository, commit: commit, dir: dir}
|
||||
if repository == "" || commit == "" {
|
||||
ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module)
|
||||
t.why = "the controller's record names no repository or commit"
|
||||
return t, nil
|
||||
}
|
||||
// One try for each repository, commit and directory: a repository with no manifest where it was registered is said once,
|
||||
// not at every record of the controller's builds. A person's take-in (force) reads again.
|
||||
tried := repository + "@" + commit + ":" + dir
|
||||
ix.mu.Lock()
|
||||
if ix.tried[tried] && !force {
|
||||
ix.mu.Unlock()
|
||||
t.why = "tried already at this commit and directory"
|
||||
return t, nil
|
||||
}
|
||||
if ix.tried == nil {
|
||||
ix.tried = map[string]bool{}
|
||||
}
|
||||
ix.tried[tried] = true
|
||||
ix.mu.Unlock()
|
||||
read, err := ix.Read(ctx, repository, commit, dir)
|
||||
if err != nil {
|
||||
ix.mu.Lock()
|
||||
delete(ix.tried, tried) // a forge that could not say is asked again
|
||||
if ix.unread == nil {
|
||||
ix.unread = map[string]bool{}
|
||||
}
|
||||
ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread)
|
||||
ix.mu.Unlock()
|
||||
return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err)
|
||||
}
|
||||
ix.markRead(r.Module)
|
||||
raw, ok := read.Manifests[dir]
|
||||
if !ok || read.Trees[dir] == "" {
|
||||
ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir)
|
||||
t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir)
|
||||
return t, nil
|
||||
}
|
||||
if name := manifestModule(raw); name != r.Module {
|
||||
ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name)
|
||||
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
|
||||
return t, nil
|
||||
}
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]},
|
||||
Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{dir: l}
|
||||
}
|
||||
if err := ix.applyManifests(ctx, f, false); err != nil {
|
||||
return t, err
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return t, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if !found {
|
||||
t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)"
|
||||
return t, nil
|
||||
}
|
||||
t.stored = true
|
||||
ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// markRead takes a module out of the unread ones: the forge answered for it.
|
||||
func (ix *Indexer) markRead(module string) {
|
||||
ix.mu.Lock()
|
||||
delete(ix.unread, module)
|
||||
ix.mu.Unlock()
|
||||
}
|
||||
|
||||
// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while
|
||||
// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the
|
||||
// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand
|
||||
// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next.
|
||||
func (ix *Indexer) RetryUnread(ctx context.Context) {
|
||||
ix.mu.Lock()
|
||||
names := slices.Sorted(maps.Keys(ix.unread))
|
||||
ix.mu.Unlock()
|
||||
for _, module := range names {
|
||||
if err := ix.IndexRegisteredModule(ctx, module); err != nil {
|
||||
ix.logf("the registered module %s is still not indexed: %v", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without
|
||||
// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run
|
||||
// once more when it ends; never two at once.
|
||||
func (ix *Indexer) kickRetry(ctx context.Context) {
|
||||
ix.mu.Lock()
|
||||
none := len(ix.unread) == 0
|
||||
ix.mu.Unlock()
|
||||
if none {
|
||||
return
|
||||
}
|
||||
ix.retryAgain.Store(true)
|
||||
if !ix.retrying.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
ix.retryWG.Add(1)
|
||||
go func() {
|
||||
defer ix.retryWG.Done()
|
||||
for ix.retryAgain.Swap(false) {
|
||||
ix.RetryUnread(ctx)
|
||||
}
|
||||
ix.retrying.Store(false)
|
||||
// A kick between the last Swap and the Store found it running and returned: run for it.
|
||||
if ix.retryAgain.Load() {
|
||||
ix.kickRetry(ctx)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from
|
||||
// where, and otherwise why not.
|
||||
type TakenIn struct {
|
||||
Module string `json:"module"`
|
||||
Stored bool `json:"stored"`
|
||||
Held bool `json:"held,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
|
||||
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
|
||||
// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the
|
||||
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
|
||||
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
if ix.Builds == nil {
|
||||
out.Answer = "the indexer reads no controller's build records, so it knows no registration"
|
||||
return out
|
||||
}
|
||||
regs := ix.Builds.RegistrationsOf(module)
|
||||
if len(regs) == 0 {
|
||||
if key, err := delivery.ModulesKey(module); err == nil && ix.Modules != nil {
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
|
||||
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
|
||||
return out
|
||||
}
|
||||
}
|
||||
out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge"
|
||||
return out
|
||||
}
|
||||
var whys []string
|
||||
for _, r := range regs {
|
||||
t, err := ix.indexRegistered(ctx, r, true)
|
||||
out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir)
|
||||
switch {
|
||||
case err != nil:
|
||||
whys = append(whys, err.Error())
|
||||
continue
|
||||
case t.held:
|
||||
out.Held, out.Answer = true, t.why
|
||||
out.Repository, out.Commit, out.Path = "", "", ""
|
||||
return out
|
||||
case t.stored:
|
||||
out.Stored = true
|
||||
out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit)
|
||||
return out
|
||||
}
|
||||
whys = append(whys, t.why)
|
||||
}
|
||||
out.Answer = "not taken in: " + strings.Join(whys, "; ")
|
||||
return out
|
||||
}
|
||||
|
||||
// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one.
|
||||
func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error {
|
||||
if ix.Builds == nil {
|
||||
return nil
|
||||
}
|
||||
var first error
|
||||
for _, r := range ix.Builds.RegistrationsOf(module) {
|
||||
if err := ix.IndexRegistered(ctx, r); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is
|
||||
// said and the others go on; the first error is answered.
|
||||
func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
|
||||
if ix.Builds == nil {
|
||||
return nil
|
||||
}
|
||||
var first error
|
||||
for _, r := range ix.Builds.Registered() {
|
||||
if err := ix.IndexRegistered(ctx, r); err != nil {
|
||||
ix.logf("the registered module %s is not indexed: %v", r.Module, err)
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
|
||||
const forge = "http://novox.internal:20000/jschoubben/snake.git"
|
||||
snake := `{"module":"snake","version":"1"}`
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
read RepoRead
|
||||
err error
|
||||
ref string // the build record's ref
|
||||
heads map[string]string // the gate source's heads
|
||||
wantRead bool // the forge is read
|
||||
held bool // modules.snake exists already
|
||||
other bool // modules.other of the same repository exists
|
||||
want bool // modules.snake is written
|
||||
}{
|
||||
{wantRead: true, name: "a module at the root of its repository", path: "", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a root spelled as a dot", path: ".", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a module in a directory", path: "modules/snake", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}},
|
||||
{wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{name: "a module the bucket holds is left", path: "", held: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "no manifest at the path", path: "modules/x",
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a manifest of another module", path: "",
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1"}`}}},
|
||||
{wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2},
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
|
||||
if tc.held {
|
||||
if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.other {
|
||||
if err := mods.Create(context.Background(), "other", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ix.Manifests = newMem()
|
||||
var asked struct {
|
||||
repo, sha, dir string
|
||||
n int
|
||||
}
|
||||
ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) {
|
||||
asked.repo, asked.sha, asked.dir = repo, sha, dir
|
||||
asked.n++
|
||||
return tc.read, tc.err
|
||||
}
|
||||
if tc.heads != nil {
|
||||
ix.source = sourceNow{rev: 1, heads: tc.heads}
|
||||
}
|
||||
err := ix.IndexRegisteredAll(context.Background())
|
||||
if (err != nil) != (tc.err != nil) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's
|
||||
if tc.heads != nil {
|
||||
wantCommit = h2 // the source's head
|
||||
}
|
||||
if tc.wantRead && asked.sha != wantCommit {
|
||||
t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit)
|
||||
}
|
||||
wantReads := 1
|
||||
if !tc.wantRead {
|
||||
wantReads = 0
|
||||
}
|
||||
if tc.wantRead && asked.dir != dirOf(tc.path) {
|
||||
t.Errorf("the forge was asked for directory %q", asked.dir)
|
||||
}
|
||||
if asked.n != wantReads {
|
||||
t.Errorf("forge reads = %d, want %d", asked.n, wantReads)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "snake")
|
||||
if tc.want {
|
||||
if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) {
|
||||
t.Fatalf("modules.snake = %+v, found %v", m, ok)
|
||||
}
|
||||
if len(ix.Manifests.(*memStore).vals) != 1 {
|
||||
t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals))
|
||||
}
|
||||
if asked.repo != "jschoubben/snake" {
|
||||
t.Errorf("the forge was asked for %q", asked.repo)
|
||||
}
|
||||
} else if !tc.held && ok {
|
||||
t.Fatalf("modules.snake written: %+v", m)
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "other"); ok != tc.other {
|
||||
t.Errorf("modules.other held = %v, want %v", ok, tc.other)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustModule(t *testing.T, m delivery.Module) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeModule(withV(m))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func dirOf(p string) string {
|
||||
if p == "." {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh
|
||||
const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git"
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""),
|
||||
rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""),
|
||||
rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", ""))
|
||||
ix.Manifests = newMem()
|
||||
var asked []string
|
||||
ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) {
|
||||
asked = append(asked, repo+":"+dir)
|
||||
if repo == "novox/Link2Pay" {
|
||||
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1"}`}}, nil
|
||||
}
|
||||
return RepoRead{Root: "r0"}, nil
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "link2pay")
|
||||
if !ok || m.Repository != "novox/Link2Pay" {
|
||||
t.Fatalf("modules.link2pay = %+v, found %v", m, ok)
|
||||
}
|
||||
if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" {
|
||||
t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit
|
||||
// Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when
|
||||
// the indexer started again. Each row's trigger must now take the module in once the forge reads it.
|
||||
const snake = `{"module":"snake","version":"1"}`
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct{ failure, trigger string }{
|
||||
{"timeout", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a new source revision"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a merged message of another repository"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
|
||||
{"timeout", "a person's take-in"},
|
||||
} {
|
||||
t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", ""))
|
||||
ix.Manifests = newMem()
|
||||
ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}}
|
||||
var asked []string
|
||||
readable := false
|
||||
ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) {
|
||||
asked = append(asked, sha)
|
||||
if readable {
|
||||
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil
|
||||
}
|
||||
return RepoRead{}, errors.New(tc.failure)
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(ctx); err == nil {
|
||||
t.Errorf("no error")
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 {
|
||||
t.Errorf("written %v, asked %v", ok, asked)
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 {
|
||||
t.Errorf("not asked again (asked %v)", asked)
|
||||
}
|
||||
readable = true // the forge's account reads the repository now
|
||||
var got TakenIn
|
||||
switch tc.trigger {
|
||||
case "the indexer starts again":
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "a new source revision":
|
||||
if err := ix.ApplySource(ctx, 2, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait() // read again in the background
|
||||
case "a merged message of another repository":
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait() // read again in the background
|
||||
case "a person's take-in":
|
||||
got = ix.TakeIn(ctx, "snake")
|
||||
if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" {
|
||||
t.Errorf("take-in answered %+v", got)
|
||||
}
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 {
|
||||
t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked)
|
||||
}
|
||||
// Taken in, it is read no more: a further event asks the forge nothing.
|
||||
n := len(asked)
|
||||
if err := ix.ApplySource(ctx, 3, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait()
|
||||
if len(asked) != n {
|
||||
t.Errorf("the forge was asked again after the module was taken in: %v", asked)
|
||||
}
|
||||
if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n {
|
||||
t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked)
|
||||
}
|
||||
if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") {
|
||||
t.Errorf("a take-in of an unregistered module answered %+v", none)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place
|
||||
const repo = "http://novox.internal:20000/novox/photos.git"
|
||||
b := NewBuilds()
|
||||
for _, r := range [][2]any{
|
||||
rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""),
|
||||
rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""),
|
||||
rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""),
|
||||
} {
|
||||
b.Put(r[0].(string), r[1].([]byte))
|
||||
}
|
||||
got := b.RegistrationsOf("photos")
|
||||
if len(got) != 1 || got[0].Ref != "nox-mesh" {
|
||||
t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got)
|
||||
}
|
||||
k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "")
|
||||
b.Put(k, v) // a trunk's ref beats a branch of its own
|
||||
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
|
||||
t.Fatalf("registrations = %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,10 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
|
||||
if ix.Modules == nil {
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
|
||||
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
|
||||
// (applyManifests reads srcOf).
|
||||
ix.kickRetry(ctx)
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
|
||||
// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool
|
||||
// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's
|
||||
// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools
|
||||
// of the indexer's states stay in mesh-delivery-view.
|
||||
const (
|
||||
TakeInTool = "indexer_take_in"
|
||||
// PlacementRereadTool reads the catalogue's placement again now (ADR 0559 §8); it asks through the same state, under
|
||||
// PlacementRereadKey, a key no module has (a module's name holds no dot).
|
||||
PlacementRereadTool = "indexer_placement_reread"
|
||||
PlacementRereadKey = "placement.reread"
|
||||
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
|
||||
TakeInsState = "takeins"
|
||||
TakeInsBucket = delivery.IndexerModule + "_" + TakeInsState
|
||||
)
|
||||
|
||||
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
|
||||
// indexer read it, its answer.
|
||||
type TakeInAsk struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
// Placement asks indexer_placement_reread instead of a module's take-in; Reread is its answer.
|
||||
Placement bool `json:"placement,omitempty"`
|
||||
Reread *PlacementReread `json:"reread,omitempty"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
func (a TakeInAsk) Done() bool { return !a.Answered.IsZero() && (a.Result != nil || a.Reread != nil) }
|
||||
|
||||
// NewPlacementReread is the value indexer_placement_reread writes: its key and the value.
|
||||
func NewPlacementReread(why, by string, at time.Time) (string, []byte, error) {
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Placement: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return PlacementRereadKey, data, err
|
||||
}
|
||||
|
||||
// DecodeTakeIn reads a takeins value.
|
||||
func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
var a TakeInAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" && !a.Placement {
|
||||
return a, errors.New("a takeins value naming no module")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// NewTakeIn is the value indexer_take_in writes for a module: its key and the value.
|
||||
func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
|
||||
module = strings.TrimSpace(module)
|
||||
if module == "" {
|
||||
return "", nil, errors.New("name the module to take in (module)")
|
||||
}
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// What the indexer took in (novox/hq ADR 0556 clause 1, issue 1280): after a merged message is applied whole, the key of
|
||||
// its repository in the state `taken` names that merge, so a reader of the modules knows they say the repository at that
|
||||
// merge or later. The indexer is the key's only writer. A merge heard again after a later one (a redelivery) does not put
|
||||
// the key back: the key moves only to a merge that contains the one it names, as the forge says it; a forge that cannot
|
||||
// say keeps the key as it is and says so.
|
||||
|
||||
// writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one.
|
||||
func (ix *Indexer) writeTaken(ctx context.Context, f delivery.Forge) error {
|
||||
if ix.Taken == nil || f.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.RepositoryKey(f.Repository)
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: no taken key is written: %v", f.Repository, f.Merge, err)
|
||||
return nil
|
||||
}
|
||||
err = write(ctx, ix.Taken, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeTaken(value); err == nil {
|
||||
if cur.Merge == f.Merge {
|
||||
return nil, false, nil
|
||||
}
|
||||
if ix.Contains != nil {
|
||||
later, err := ix.Contains(ctx, f.Repository, f.Merge, cur.Merge)
|
||||
if err != nil {
|
||||
ix.logf("%s: whether %s is in %s cannot be read, so taken.%s stays at %s: %v", f.Repository, f.Merge, cur.Merge, key, cur.Merge, err)
|
||||
return nil, false, nil
|
||||
}
|
||||
if later {
|
||||
return nil, false, nil // a merge heard again after a later one
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
now := time.Now
|
||||
if ix.Now != nil {
|
||||
now = ix.Now
|
||||
}
|
||||
data, err := delivery.EncodeTaken(delivery.Taken{Repository: delivery.RepositoryName(f.Repository), Merge: f.Merge, At: now().UTC()})
|
||||
return data, false, err
|
||||
})
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
// The state is not on the bus yet (a delivery that adds it, bound when it appears as issue 1790 says): said, and
|
||||
// the merge's modules are still taken in; FillTaken writes its key once the state appears.
|
||||
ix.logf("%s at %s: its taken key is not written yet, FillTaken writes it when the state appears: %v", f.Repository, f.Merge, err)
|
||||
ix.mu.Lock()
|
||||
if ix.untaken == nil {
|
||||
ix.untaken = map[string]string{}
|
||||
}
|
||||
ix.untaken[f.Repository] = f.Merge // the latest heard; FillTaken keeps a later key over an earlier merge
|
||||
ix.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// FillTaken writes the taken keys that were left unwritten, never a merge the indexer has not applied whole (a key names
|
||||
// a merge only after it was applied: ADR 0556 clause 1, and the view reads it as the upper bound of what is reached). Two
|
||||
// sources, both through writeTaken as a live merge does (issue 1280):
|
||||
// - the merges applied while the state was not on the bus (kept by writeTaken), whenever the state is there;
|
||||
// - the gate's source heads, only when Drained says the indexer has applied every message of the forge stream that was
|
||||
// there before the heads were read. Otherwise it says so, and the next kick of the fill (after a message is applied)
|
||||
// tries again.
|
||||
//
|
||||
// It is idempotent: a key naming the merge or a later one is left. A state not on the bus yet writes nothing and says
|
||||
// nothing (the fill runs again when it appears). The first error is returned after every repository was tried.
|
||||
func (ix *Indexer) FillTaken(ctx context.Context) error {
|
||||
if ix.Taken == nil {
|
||||
return nil
|
||||
}
|
||||
if _, err := ix.Taken.Keys(ctx); errors.Is(err, process.ErrNotYet) {
|
||||
return nil
|
||||
}
|
||||
ix.mu.Lock()
|
||||
pending := maps.Clone(ix.untaken)
|
||||
heads := maps.Clone(ix.source.heads)
|
||||
ix.mu.Unlock()
|
||||
var first error
|
||||
note := func(repository string, err error) {
|
||||
if err != nil && first == nil {
|
||||
first = fmt.Errorf("%s: %w", repository, err)
|
||||
}
|
||||
}
|
||||
for _, repository := range slices.Sorted(maps.Keys(pending)) {
|
||||
err := ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: pending[repository]})
|
||||
if err == nil {
|
||||
ix.mu.Lock()
|
||||
if ix.untaken[repository] == pending[repository] {
|
||||
delete(ix.untaken, repository)
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
}
|
||||
note(repository, err)
|
||||
}
|
||||
if ix.Drained == nil {
|
||||
return first
|
||||
}
|
||||
if len(heads) == 0 {
|
||||
ix.logf("the source names no repository head, so no taken key is filled from it")
|
||||
return first
|
||||
}
|
||||
done, err := ix.Drained(ctx)
|
||||
if err != nil {
|
||||
ix.logf("whether the forge stream is applied cannot be read, so no taken key is filled from the source: %v", err)
|
||||
return first
|
||||
}
|
||||
if !done {
|
||||
ix.logf("the forge stream is not applied whole yet, so no taken key is filled from the source; the next applied message tries again")
|
||||
return first
|
||||
}
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
note(repository, ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: heads[repository]}))
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// KickFill runs FillTaken in the background after a message was applied, one at a time; a kick that arrives while one runs
|
||||
// makes it run once more.
|
||||
func (ix *Indexer) KickFill(ctx context.Context) {
|
||||
if ix.filling.Swap(true) {
|
||||
ix.fillAgain.Store(true)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
ix.fillAgain.Store(false)
|
||||
if err := ix.FillTaken(ctx); err != nil {
|
||||
ix.logf("filling the taken keys: %v", err)
|
||||
}
|
||||
if !ix.fillAgain.Load() {
|
||||
ix.filling.Store(false)
|
||||
if !ix.fillAgain.Load() || ix.filling.Swap(true) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user