Compare commits
101
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
feef96ed16 | ||
|
|
4f722899df | ||
|
|
166a918b43 | ||
|
|
1b4c860f63 | ||
|
|
3b6945b2db | ||
|
|
4174e23c04 | ||
|
|
3993175e45 | ||
|
|
60c7ffc981 | ||
|
|
33de9dde4f | ||
|
|
6b85220fa4 | ||
|
|
1cbae7d41c | ||
|
|
b3a4b09f5b | ||
|
|
c8b26735c8 | ||
|
|
907e092abc | ||
|
|
3073495a06 | ||
|
|
16c411e436 | ||
|
|
2d841870a8 | ||
|
|
7f8ce7f5c0 | ||
|
|
07b5f13474 | ||
|
|
f4aac599ca | ||
|
|
2d63f0ba62 | ||
|
|
1d41682715 | ||
|
|
705765d113 | ||
|
|
16a97ed65d | ||
|
|
ecf532af0e | ||
|
|
ce600df98a | ||
|
|
b4a6a34705 | ||
|
|
a7e51b416b | ||
|
|
71f786f146 | ||
|
|
848c6fdc56 | ||
|
|
ead9ee2d13 | ||
|
|
4861d1f78d | ||
|
|
5c76c9d3ac | ||
|
|
699cbfb7c1 | ||
|
|
319d56be7e | ||
|
|
11d3678984 | ||
|
|
f55f150896 | ||
|
|
e1f1b5d18b | ||
|
|
f0fe454145 | ||
|
|
4026f6c188 | ||
|
|
29cfb8c177 | ||
|
|
a092ee9ff9 | ||
|
|
6c3cbee35c | ||
|
|
4b23db28b3 | ||
|
|
646f83642a | ||
|
|
d9bfba1e62 | ||
|
|
f1bb7747ae | ||
|
|
d2eb3e5ecf | ||
|
|
b0b1cf4f01 | ||
|
|
d91ea41f64 | ||
|
|
5fed429827 | ||
|
|
9a569bfb8d | ||
|
|
e2c41ffa5a | ||
|
|
13e4ed8515 | ||
|
|
c66fe11e16 | ||
|
|
75d0eb32be | ||
|
|
c44b9f91cd | ||
|
|
82da252cf0 | ||
|
|
ce72b67cef | ||
|
|
32bc4f3c48 | ||
|
|
2a94d16834 | ||
|
|
da7baa5e39 | ||
|
|
ed1125b66d | ||
|
|
366bc750da | ||
|
|
04a56b7aff | ||
|
|
2f697216ee | ||
|
|
769dab43a2 | ||
|
|
98a05997ae | ||
|
|
07dee528c5 | ||
|
|
7ddbf40715 | ||
|
|
aea22bf793 | ||
|
|
72889be87f | ||
|
|
86db64904b | ||
|
|
12f16bbebd | ||
|
|
4ec40eec9a | ||
|
|
13380e3d0e | ||
|
|
9132b6a296 | ||
|
|
bcb5aafdbe | ||
|
|
3e8bd9b0de | ||
|
|
b842b9422d | ||
|
|
9db15c7cf4 | ||
|
|
ba47b47e24 | ||
|
|
b566563dc5 | ||
|
|
2a856e2a76 | ||
|
|
ce7d60cb6e | ||
|
|
26ba7404bf | ||
|
|
b4af9cdb7a | ||
|
|
db53ec975f | ||
|
|
1ff5049d03 | ||
|
|
41f25d3129 | ||
|
|
4628ba26b4 | ||
|
|
20027e53ff | ||
|
|
d4a054bfed | ||
|
|
c623e8d235 | ||
|
|
998fe3a515 | ||
|
|
73be3df28c | ||
|
|
eb65535f45 | ||
|
|
fa6cdbbe61 | ||
|
|
462ec9cfb1 | ||
|
|
9950689a11 | ||
|
|
71a6a5bf77 |
@@ -182,6 +182,42 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
|||||||
t.Errorf("without %s the assigner started", missing)
|
t.Errorf("without %s the assigner started", missing)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
|
||||||
|
// ADR 0201) a pass opens it, so a put is taken further without a restart.
|
||||||
|
was := PutsReopenEvery
|
||||||
|
PutsReopenEvery = 0
|
||||||
|
defer func() { PutsReopenEvery = was }()
|
||||||
|
b := startBed(t, delivery.AssignerPutsBucket)
|
||||||
|
done := b.start(t)
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
}
|
||||||
|
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pkey, _ := delivery.PutKey("n1", "web")
|
||||||
|
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
|
||||||
|
raw, _ := delivery.EncodePut(asked)
|
||||||
|
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||||
|
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
|
||||||
|
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if e, err := puts.Get(b.ctx, pkey); err == nil {
|
||||||
|
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("the put was not taken further once its bucket was made")
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// putJSON writes a value into one of the bed's buckets.
|
// putJSON writes a value into one of the bed's buckets.
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"os/signal"
|
"os/signal"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
@@ -85,11 +86,11 @@ var keptStates = []announce.State{
|
|||||||
// buckets is every bucket the assigner opened, by name.
|
// buckets is every bucket the assigner opened, by name.
|
||||||
type buckets map[string]jetstream.KeyValue
|
type buckets map[string]jetstream.KeyValue
|
||||||
|
|
||||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
|
||||||
// without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
||||||
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
||||||
// without it and takes no put further until it starts again (ADR 0546).
|
// without it and takes no put further until it starts again (ADR 0546).
|
||||||
func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) {
|
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||||
out := buckets{}
|
out := buckets{}
|
||||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||||
kv, err := js.KeyValue(ctx, name)
|
kv, err := js.KeyValue(ctx, name)
|
||||||
@@ -98,7 +99,7 @@ func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||||
}
|
}
|
||||||
out[name] = kv
|
out[name] = kv
|
||||||
}
|
}
|
||||||
@@ -213,7 +214,9 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if e == nil {
|
if e == nil {
|
||||||
settled <- struct{}{}
|
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
|
||||||
|
settled <- struct{}{}
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var value []byte
|
var value []byte
|
||||||
@@ -227,15 +230,39 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
}()
|
}()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
followed := watched
|
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
|
||||||
|
// stops the puts, never the wants.
|
||||||
if kvs[delivery.AssignerPutsBucket] != nil {
|
if kvs[delivery.AssignerPutsBucket] != nil {
|
||||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||||
logger.Printf("%v: no put is taken further until the assigner starts again", err)
|
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||||
delete(kvs, delivery.AssignerPutsBucket)
|
delete(kvs, delivery.AssignerPutsBucket)
|
||||||
} else {
|
|
||||||
followed = append(append([]string{}, watched...), delivery.AssignerPutsBucket)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
|
||||||
|
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
|
||||||
|
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
|
||||||
|
// further, without a restart.
|
||||||
|
var triedPuts time.Time
|
||||||
|
reopenPuts := func(p *passer) {
|
||||||
|
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
triedPuts = time.Now()
|
||||||
|
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kvs[delivery.AssignerPutsBucket] = kv
|
||||||
|
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||||
|
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||||
|
delete(kvs, delivery.AssignerPutsBucket)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if bus != nil {
|
||||||
|
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||||
|
}
|
||||||
|
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||||
|
}
|
||||||
for _, name := range watched {
|
for _, name := range watched {
|
||||||
var opts []jetstream.WatchOpt
|
var opts []jetstream.WatchOpt
|
||||||
if name == delivery.HaveBucket {
|
if name == delivery.HaveBucket {
|
||||||
@@ -248,7 +275,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for range len(followed) + 1 {
|
for range len(watched) + 1 {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil
|
return nil
|
||||||
@@ -271,11 +298,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
return err
|
return err
|
||||||
case <-settled:
|
case <-settled:
|
||||||
case <-dirty:
|
case <-dirty:
|
||||||
|
reopenPuts(p)
|
||||||
p.pass(ctx)
|
p.pass(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||||
|
var PutsReopenEvery = time.Minute
|
||||||
|
|
||||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||||
// (ADR 0437 clause 9).
|
// (ADR 0437 clause 9).
|
||||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||||
|
|||||||
@@ -5,15 +5,18 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -70,10 +73,36 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
|||||||
switch {
|
switch {
|
||||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)):
|
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
|
||||||
|
// keep the first pass from being made: the wants are written all the same.
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
wrote := make(chan string, 8)
|
||||||
|
kvs := buckets{}
|
||||||
|
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||||
|
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||||
|
}
|
||||||
|
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||||
|
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||||
|
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||||
|
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||||
|
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||||
|
select {
|
||||||
|
case key := <-wrote:
|
||||||
|
if key != "want.ace.node" {
|
||||||
|
t.Errorf("the first pass wrote %s", key)
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1297/B2a: without a credential the program does not start, and says why.
|
// 1297/B2a: without a credential the program does not start, and says why.
|
||||||
@@ -97,3 +126,47 @@ func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue
|
|||||||
}
|
}
|
||||||
return &memKV{values: map[string][]byte{}}, nil
|
return &memKV{values: map[string][]byte{}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||||
|
// closed; a put on it is said on wrote.
|
||||||
|
type watchKV struct {
|
||||||
|
*memKV
|
||||||
|
entries []kvEntry
|
||||||
|
closed bool
|
||||||
|
wrote chan string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||||
|
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
|
||||||
|
if w.closed {
|
||||||
|
close(ch)
|
||||||
|
return keyWatcher{ch}, nil
|
||||||
|
}
|
||||||
|
for _, e := range w.entries {
|
||||||
|
ch <- e
|
||||||
|
}
|
||||||
|
ch <- nil
|
||||||
|
return keyWatcher{ch}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||||
|
if w.wrote != nil {
|
||||||
|
w.wrote <- key
|
||||||
|
}
|
||||||
|
return 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
|
||||||
|
|
||||||
|
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
|
||||||
|
func (k keyWatcher) Stop() error { return nil }
|
||||||
|
|
||||||
|
type kvEntry struct {
|
||||||
|
jetstream.KeyValueEntry
|
||||||
|
key string
|
||||||
|
value []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e kvEntry) Key() string { return e.key }
|
||||||
|
func (e kvEntry) Value() []byte { return e.value }
|
||||||
|
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||||
|
|||||||
Binary file not shown.
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -35,6 +36,9 @@ type passer struct {
|
|||||||
neither map[string]neitherOne
|
neither map[string]neitherOne
|
||||||
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
||||||
acts *putActs
|
acts *putActs
|
||||||
|
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||||
|
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||||
|
noWant map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *passer) clock() time.Time {
|
func (p *passer) clock() time.Time {
|
||||||
@@ -94,6 +98,7 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
wanted := p.w.wantsOf()
|
wanted := p.w.wantsOf()
|
||||||
puts := p.putsByModule()
|
puts := p.putsByModule()
|
||||||
|
p.staleAsks(ctx, trunk)
|
||||||
names := map[string]bool{}
|
names := map[string]bool{}
|
||||||
for name := range trunk {
|
for name := range trunk {
|
||||||
names[name] = true
|
names[name] = true
|
||||||
@@ -108,7 +113,8 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
// quiet at the router.
|
// quiet at the router.
|
||||||
first := p.neither == nil
|
first := p.neither == nil
|
||||||
stale := map[string]neitherOne{}
|
stale := map[string]neitherOne{}
|
||||||
defer func() { p.sayNeither(neither, stale) }()
|
left := map[string]string{}
|
||||||
|
defer func() { p.sayNeither(neither, stale, left) }()
|
||||||
for name := range names {
|
for name := range names {
|
||||||
fp := trunk[name].FP2
|
fp := trunk[name].FP2
|
||||||
name := name
|
name := name
|
||||||
@@ -148,6 +154,8 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
out := assign.Wants(m, nodes, wanted[name], last)
|
out := assign.Wants(m, nodes, wanted[name], last)
|
||||||
gatherNeither(neither, name, out)
|
gatherNeither(neither, name, out)
|
||||||
|
leftNeither(left, m, wanted[name], out)
|
||||||
|
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||||
if first {
|
if first {
|
||||||
startClears(stale, m, out)
|
startClears(stale, m, out)
|
||||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||||
@@ -186,6 +194,42 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
|
||||||
|
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
|
||||||
|
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
|
||||||
|
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
|
||||||
|
if p.noWant == nil {
|
||||||
|
p.noWant = map[string]string{}
|
||||||
|
}
|
||||||
|
assigned := map[string]bool{}
|
||||||
|
for _, node := range nodes {
|
||||||
|
assigned[node] = true
|
||||||
|
key := node + "." + module
|
||||||
|
w, written := out.Wants[node]
|
||||||
|
if !written {
|
||||||
|
w, written = before[node]
|
||||||
|
}
|
||||||
|
reason := ""
|
||||||
|
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||||
|
reason = h.Reason()
|
||||||
|
}
|
||||||
|
last, said := p.noWant[key]
|
||||||
|
switch {
|
||||||
|
case reason != "" && reason != last:
|
||||||
|
p.logf("%s on %s gets no want: %s", module, node, reason)
|
||||||
|
p.noWant[key] = reason
|
||||||
|
case reason == "" && said && written && !w.Removed:
|
||||||
|
p.logf("%s on %s has a want", module, node)
|
||||||
|
delete(p.noWant, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key := range p.noWant {
|
||||||
|
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
|
||||||
|
delete(p.noWant, key) // no longer assigned there: its removed want says so
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||||
type neitherOne struct {
|
type neitherOne struct {
|
||||||
node, module string
|
node, module string
|
||||||
@@ -199,6 +243,48 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||||
|
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||||
|
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||||
|
// left it with none).
|
||||||
|
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||||
|
for _, node := range m.Nodes {
|
||||||
|
if _, on := out.Neither[node]; on {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w, written := out.Wants[node]
|
||||||
|
if !written {
|
||||||
|
w, written = before[node]
|
||||||
|
}
|
||||||
|
why := ""
|
||||||
|
h, ruled := out.Rules[node]
|
||||||
|
switch {
|
||||||
|
case written && !w.Removed:
|
||||||
|
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||||
|
why = "it is held, and its held condition says so"
|
||||||
|
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||||
|
why = "its move waits for the operator's yes on its record ask"
|
||||||
|
case putOn(m, node):
|
||||||
|
why = "its put waits on its own steps and the operator's answer"
|
||||||
|
case m.FP == "":
|
||||||
|
why = "the trunk has no build of it"
|
||||||
|
case ruled:
|
||||||
|
why = h.Reason()
|
||||||
|
default:
|
||||||
|
why = "the node does not run it"
|
||||||
|
}
|
||||||
|
into[node+"."+m.Name] = why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||||
|
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||||
|
func putOn(m assign.Module, node string) bool {
|
||||||
|
p, put := m.Puts[node]
|
||||||
|
_, owns := assign.PutOwns(p)
|
||||||
|
return put && owns
|
||||||
|
}
|
||||||
|
|
||||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||||
@@ -215,7 +301,9 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
|
|||||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||||
// each is cleared once.
|
// each is cleared once.
|
||||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||||
|
// it does not hold is no longer assigned.
|
||||||
|
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||||
if p.neither == nil {
|
if p.neither == nil {
|
||||||
p.neither = map[string]neitherOne{}
|
p.neither = map[string]neitherOne{}
|
||||||
for key, one := range stale {
|
for key, one := range stale {
|
||||||
@@ -235,21 +323,29 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
one.module, one.node, one.holding.Reason())
|
||||||
p.neither[key] = one
|
p.neither[key] = one
|
||||||
}
|
}
|
||||||
for key, one := range p.neither {
|
for key, one := range p.neither {
|
||||||
if _, still := now[key]; still {
|
if _, still := now[key]; still {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
why, assigned := left[key]
|
||||||
|
if !assigned {
|
||||||
|
why = "it is no longer assigned there"
|
||||||
|
}
|
||||||
if p.say != nil {
|
if p.say != nil {
|
||||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p.logf("%s on %s has a want again", one.module, one.node)
|
if why == "" {
|
||||||
|
p.logf("%s on %s has a want again", one.module, one.node)
|
||||||
|
} else {
|
||||||
|
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||||
|
}
|
||||||
delete(p.neither, key)
|
delete(p.neither, key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -363,6 +459,98 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
|
|||||||
}
|
}
|
||||||
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
||||||
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
||||||
|
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
|
||||||
|
// set back from it (issue 1906, note 28).
|
||||||
|
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||||
|
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
|
||||||
|
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
|
||||||
|
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
|
||||||
|
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
|
||||||
|
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
|
||||||
|
// nothing taken away.
|
||||||
|
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
|
||||||
|
k, err := delivery.ParseAnyAskKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch k.Kind {
|
||||||
|
case delivery.AskKindHold, delivery.AskKindRestore:
|
||||||
|
m, on := trunk[k.Module]
|
||||||
|
return !on || (m.FP2 != "" && m.FP2 != k.FP)
|
||||||
|
case delivery.AskKindRecord:
|
||||||
|
m, on := trunk[k.Module]
|
||||||
|
if !on {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
|
||||||
|
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
|
||||||
|
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
|
||||||
|
// trunk's, its want's, its have's), and only when every one of them is read.
|
||||||
|
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
|
||||||
|
d, ok := p.w.composed(k.Node, k.Module, fp)
|
||||||
|
if !ok || d.Digest == k.Composed {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
|
||||||
|
// names, each once.
|
||||||
|
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
|
||||||
|
var out []string
|
||||||
|
add := func(fp string) {
|
||||||
|
if fp != "" && !slices.Contains(out, fp) {
|
||||||
|
out = append(out, fp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
add(trunkFP)
|
||||||
|
if key, err := delivery.WantKey(node, module); err == nil {
|
||||||
|
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
|
||||||
|
add(w.FP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if key, err := delivery.HaveKey(node, module); err == nil {
|
||||||
|
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
|
||||||
|
add(h.FP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
|
||||||
|
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
|
||||||
|
// read at all (no module), nothing is taken away.
|
||||||
|
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
|
||||||
|
if len(trunk) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||||
|
if p.staleAsk(key, trunk) {
|
||||||
|
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
|
||||||
|
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
|
||||||
|
k, kerr := delivery.ParseAnyAskKey(key)
|
||||||
|
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
|
||||||
|
if held && !p.clearHeld(k.Module, k.FP) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
|
||||||
|
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||||
|
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
|
||||||
|
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
|
||||||
|
// raised again if the trunk returns to the fingerprint.
|
||||||
|
if held {
|
||||||
|
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
||||||
@@ -437,16 +625,18 @@ func (p *passer) answeredWith(key, option string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// clearHeld clears, once, the condition of a hold the operator released.
|
// clearHeld clears, once, the condition of a hold the operator released.
|
||||||
func (p *passer) clearHeld(module, fp string) {
|
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
|
||||||
|
func (p *passer) clearHeld(module, fp string) bool {
|
||||||
c := assign.HeldCondition(module, fp, "", "", false, false)
|
c := assign.HeldCondition(module, fp, "", "", false, false)
|
||||||
if p.say == nil || p.said[c.Key+".cleared"] {
|
if p.say == nil || p.said[c.Key+".cleared"] {
|
||||||
return
|
return true
|
||||||
}
|
}
|
||||||
if err := p.say(c); err != nil {
|
if err := p.say(c); err != nil {
|
||||||
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||||
return
|
return false
|
||||||
}
|
}
|
||||||
p.said[c.Key+".cleared"] = true
|
p.said[c.Key+".cleared"] = true
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -356,20 +357,37 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
|||||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||||
p.sayNeither(one, nil) // the say fails
|
p.sayNeither(one, nil, nil) // the say fails
|
||||||
p.sayNeither(one, nil)
|
p.sayNeither(one, nil, nil)
|
||||||
p.sayNeither(one, nil)
|
p.sayNeither(one, nil, nil)
|
||||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||||
}
|
}
|
||||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||||
}
|
}
|
||||||
p.sayNeither(map[string]neitherOne{}, nil)
|
wants := map[string]string{"g14.screen-lock": ""}
|
||||||
p.sayNeither(map[string]neitherOne{}, nil)
|
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||||
|
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||||
}
|
}
|
||||||
|
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||||
|
t.Errorf("the clear said %q", last)
|
||||||
|
}
|
||||||
|
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
|
||||||
|
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
|
||||||
|
p.sayNeither(bus, nil, nil)
|
||||||
|
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
|
||||||
|
t.Errorf("the bus's neither condition = %+v", c)
|
||||||
|
}
|
||||||
|
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||||
|
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||||
|
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||||
|
}
|
||||||
|
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||||
|
t.Errorf("the clear without a want said %q", last)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||||
@@ -380,6 +398,51 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
|||||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||||
t.Errorf("gathered = %+v", got)
|
t.Errorf("gathered = %+v", got)
|
||||||
}
|
}
|
||||||
|
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||||
|
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
puts map[string]delivery.Put
|
||||||
|
before map[string]delivery.Want
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||||
|
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||||
|
{"nothing", nil, nil, "the node does not run it"},
|
||||||
|
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||||
|
} {
|
||||||
|
left := map[string]string{}
|
||||||
|
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||||
|
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||||
|
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||||
|
// when it has a want again: one line per change, not per pass.
|
||||||
|
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||||
|
var logged []string
|
||||||
|
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||||
|
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
|
||||||
|
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
|
||||||
|
for i, c := range []struct {
|
||||||
|
before map[string]delivery.Want
|
||||||
|
out assign.Outcome
|
||||||
|
line string // the line said in this pass, "" for none
|
||||||
|
}{
|
||||||
|
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
|
||||||
|
{nil, notDone, ""},
|
||||||
|
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
|
||||||
|
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||||
|
{nil, wanted, "rofi on shanks has a want"},
|
||||||
|
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||||
|
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
|
||||||
|
} {
|
||||||
|
logged = nil
|
||||||
|
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||||
|
if got := strings.Join(logged, "\n"); got != c.line {
|
||||||
|
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||||
@@ -470,3 +533,220 @@ func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
|||||||
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
|
||||||
|
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
|
||||||
|
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
|
||||||
|
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||||
|
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
|
||||||
|
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
|
||||||
|
trunkOf := func(fp string) map[string]delivery.Module {
|
||||||
|
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
|
||||||
|
}
|
||||||
|
type ask struct{ name, key string }
|
||||||
|
holdOld, _ := delivery.HoldAskKey("web", fpOld)
|
||||||
|
holdNew, _ := delivery.HoldAskKey("web", fpNew)
|
||||||
|
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
|
||||||
|
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
|
||||||
|
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
|
||||||
|
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
|
||||||
|
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
|
||||||
|
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
|
||||||
|
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
trunk map[string]delivery.Module
|
||||||
|
gone []string // the keys taken away
|
||||||
|
noDigit bool // the controller's composition is not read
|
||||||
|
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||||
|
}{
|
||||||
|
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||||
|
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||||
|
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||||
|
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||||
|
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||||
|
{"the trunk is not read at all", nil, nil, false, ""},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||||
|
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||||
|
var said []asks.Condition
|
||||||
|
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||||
|
for _, key := range all {
|
||||||
|
k, _ := delivery.ParseAnyAskKey(key)
|
||||||
|
var a delivery.AssignerAsk
|
||||||
|
switch k.Kind {
|
||||||
|
case delivery.AskKindHold:
|
||||||
|
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
|
||||||
|
case delivery.AskKindRestore:
|
||||||
|
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
|
||||||
|
case delivery.AskKindRecord:
|
||||||
|
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
|
||||||
|
default:
|
||||||
|
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
|
||||||
|
}
|
||||||
|
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
asksKV.values[key] = v
|
||||||
|
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||||
|
}
|
||||||
|
if !c.noDigit {
|
||||||
|
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||||
|
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||||
|
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||||
|
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||||
|
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||||
|
if c.heldAt != "" {
|
||||||
|
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||||
|
}
|
||||||
|
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||||
|
}
|
||||||
|
if c.heldAt != "" {
|
||||||
|
wkey, _ := delivery.WantKey("novox", "web")
|
||||||
|
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||||
|
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||||
|
}
|
||||||
|
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||||
|
wantsKV.values[heldOld] = []byte("x")
|
||||||
|
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
|
||||||
|
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||||
|
p.staleAsks(context.Background(), c.trunk)
|
||||||
|
// the held record and condition of a fingerprint whose hold ask went are taken away with it
|
||||||
|
heldOld, _ = delivery.HeldKey("web", fpOld)
|
||||||
|
_, heldThere := wantsKV.values[heldOld]
|
||||||
|
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
|
||||||
|
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
|
||||||
|
})
|
||||||
|
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
|
||||||
|
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
|
||||||
|
}
|
||||||
|
for _, key := range all {
|
||||||
|
_, there := asksKV.values[key]
|
||||||
|
want := !slices.Contains(c.gone, key)
|
||||||
|
if there != want {
|
||||||
|
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Contains(c.gone, holdOld) {
|
||||||
|
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
|
||||||
|
p.now = func() time.Time { return time.Now().Add(time.Hour) }
|
||||||
|
p.holdAsk(context.Background(), "web", fpOld, nil)
|
||||||
|
p.heldCondition(context.Background(), "web", fpOld, nil, false)
|
||||||
|
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||||
|
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
|
||||||
|
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
|
||||||
|
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
|
||||||
|
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||||
|
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||||
|
var said []asks.Condition
|
||||||
|
fail := true
|
||||||
|
p.say = func(c asks.Condition) error {
|
||||||
|
if fail {
|
||||||
|
fail = false
|
||||||
|
return errors.New("router unreachable")
|
||||||
|
}
|
||||||
|
said = append(said, c)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
|
||||||
|
v, err := delivery.EncodeAssignerAsk(holdOld, a)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
asksKV.values[holdOld] = v
|
||||||
|
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
|
||||||
|
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||||
|
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
|
||||||
|
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
|
||||||
|
}
|
||||||
|
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||||
|
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
|
||||||
|
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
|
||||||
|
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||||
|
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||||
|
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
|
||||||
|
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
|
||||||
|
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
|
||||||
|
asksKV.values[key] = []byte("x")
|
||||||
|
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
|
||||||
|
}
|
||||||
|
p.lift(context.Background(), "web", fpOld)
|
||||||
|
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
|
||||||
|
if _, there := asksKV.values[key]; there == gone {
|
||||||
|
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustKey(key string, err error) string {
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
|
||||||
|
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||||
|
// yes the want names the revised digest.
|
||||||
|
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||||
|
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||||
|
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||||
|
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||||
|
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||||
|
p.logf = t.Logf
|
||||||
|
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||||
|
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||||
|
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||||
|
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
put(delivery.PresenceBucket, "g14", on)
|
||||||
|
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
put(delivery.HaveBucket, "g14.web", have)
|
||||||
|
wkey, _ := delivery.WantKey("g14", "web")
|
||||||
|
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||||
|
wantsKV.values[wkey] = want
|
||||||
|
put(delivery.AssignerWantsBucket, wkey, want)
|
||||||
|
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||||
|
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||||
|
if cerr != nil {
|
||||||
|
t.Fatal(cerr)
|
||||||
|
}
|
||||||
|
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||||
|
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||||
|
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||||
|
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||||
|
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
p.wants(context.Background())
|
||||||
|
if _, there := asksKV.values[akey]; !there {
|
||||||
|
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||||
|
put(delivery.AnswersBucket, akey, yes)
|
||||||
|
p.wants(context.Background())
|
||||||
|
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||||
|
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||||
|
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||||
|
}
|
||||||
|
if _, there := asksKV.values[akey]; !there {
|
||||||
|
t.Error("the answered record ask was taken away")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -96,6 +96,14 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
|||||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||||
}
|
}
|
||||||
|
if h.Bus {
|
||||||
|
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||||
|
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||||
|
Summary: headline, Headline: headline, Needs: needs,
|
||||||
|
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||||
|
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||||
|
asks.NeedsYou, needs, module, node, h.Words)}
|
||||||
|
}
|
||||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||||
Summary: headline, Headline: headline,
|
Summary: headline, Headline: headline,
|
||||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||||
@@ -103,6 +111,17 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
|||||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||||
|
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||||
|
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||||
|
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||||
|
c := NeitherCondition(node, module, Holding{}, false)
|
||||||
|
if why != "" {
|
||||||
|
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||||
func LiftedCondition(module, fp string) asks.Condition {
|
func LiftedCondition(module, fp string) asks.Condition {
|
||||||
c := HeldCondition(module, fp, "", "", false, false)
|
c := HeldCondition(module, fp, "", "", false, false)
|
||||||
|
|||||||
@@ -192,10 +192,27 @@ type RecordNeed struct {
|
|||||||
Words string
|
Words string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Holding is a rule holding a node back, and why.
|
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
|
||||||
|
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
|
||||||
|
// running an old-form build, got no want and the pass said nothing).
|
||||||
type Holding struct {
|
type Holding struct {
|
||||||
Rule int
|
Rule int
|
||||||
Words string
|
Words string
|
||||||
|
Bus bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
|
||||||
|
func (h Holding) Reason() string {
|
||||||
|
if h.Bus {
|
||||||
|
return h.Words
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
|
||||||
|
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
|
||||||
|
func BusBarWords(node, prev, fp string) string {
|
||||||
|
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||||
@@ -226,7 +243,15 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
}
|
}
|
||||||
before, had := wants[node]
|
before, had := wants[node]
|
||||||
var next delivery.Want
|
var next delivery.Want
|
||||||
if p, put := m.Puts[node]; put {
|
p, put := m.Puts[node]
|
||||||
|
if !put && had && before.Put && before.Hold {
|
||||||
|
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
||||||
|
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
||||||
|
// without hold is not kept this way: its put takes it further once heard.
|
||||||
|
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
||||||
|
Step: delivery.PutHolding}, true
|
||||||
|
}
|
||||||
|
if put {
|
||||||
if want, owns := PutOwns(p); owns {
|
if want, owns := PutOwns(p); owns {
|
||||||
if want != nil {
|
if want != nil {
|
||||||
w := *want
|
w := *want
|
||||||
@@ -295,6 +320,14 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||||
if need != nil {
|
if need != nil {
|
||||||
out.Record = append(out.Record, *need)
|
out.Record = append(out.Record, *need)
|
||||||
|
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
|
||||||
|
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
|
||||||
|
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
|
||||||
|
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
|
||||||
|
out.Rules[node] = h
|
||||||
|
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
|
||||||
|
out.Neither[node] = h
|
||||||
|
}
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -325,8 +358,18 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||||
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
|
h, there := n.Haves[node][m.Name]
|
||||||
|
switch {
|
||||||
|
case there && h.Phase != delivery.PhaseRemoved && !held:
|
||||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||||
|
case had && (!there || h.Phase == delivery.PhaseRemoved):
|
||||||
|
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
||||||
|
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
||||||
|
// would have its node agent remove the module and stop holding its composition's resources back.
|
||||||
|
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
|
||||||
|
if !delivery.SameWant(normal(before), normal(next)) {
|
||||||
|
out.Wants[node] = next
|
||||||
|
}
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -362,6 +405,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||||
|
func NoWantWords(rule int, words string) string {
|
||||||
|
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||||
|
}
|
||||||
|
|
||||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||||
// moving). A failed health while it
|
// moving). A failed health while it
|
||||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||||
@@ -443,12 +491,7 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
|||||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
if _, barred := m.busBar(before, had, running); barred {
|
||||||
prev := before.FP
|
|
||||||
if !had {
|
|
||||||
prev = running
|
|
||||||
}
|
|
||||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
|
||||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||||
}
|
}
|
||||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||||
@@ -460,6 +503,16 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
|||||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
|
||||||
|
// readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||||
|
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
|
||||||
|
prev = before.FP
|
||||||
|
if !had {
|
||||||
|
prev = running
|
||||||
|
}
|
||||||
|
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
|
||||||
|
}
|
||||||
|
|
||||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||||
|
|||||||
@@ -397,10 +397,15 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
|||||||
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||||
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||||
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||||
|
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||||
|
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||||
} {
|
} {
|
||||||
p := put
|
p := put
|
||||||
p.Step = c.step
|
p.Step = c.step
|
||||||
m.Puts = map[string]delivery.Put{"ace": p}
|
m.Puts = map[string]delivery.Put{"ace": p}
|
||||||
|
if c.step == "" {
|
||||||
|
m.Puts = nil
|
||||||
|
}
|
||||||
out := Wants(m, n, c.wanted, nil)
|
out := Wants(m, n, c.wanted, nil)
|
||||||
got, wrote := out.Wants["ace"]
|
got, wrote := out.Wants["ace"]
|
||||||
switch {
|
switch {
|
||||||
@@ -422,6 +427,23 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
|||||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||||
}
|
}
|
||||||
m.Puts = nil
|
m.Puts = nil
|
||||||
|
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
|
||||||
|
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
|
||||||
|
// condition stays raised. A node with a live want keeps it and is not on neither list.
|
||||||
|
m.Composed = composedEverywhere
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||||
|
if w, wrote := out.Wants["ace"]; wrote {
|
||||||
|
t.Errorf("the bus bar wrote ace's want %+v", w)
|
||||||
|
}
|
||||||
|
h, on := out.Neither["ace"]
|
||||||
|
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
|
||||||
|
!strings.Contains(h.Words, "assigner_put") {
|
||||||
|
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
|
||||||
|
}
|
||||||
|
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
|
||||||
|
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
|
||||||
|
}
|
||||||
stepsOfAPut(t)
|
stepsOfAPut(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -862,6 +884,33 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
|||||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||||
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
||||||
}
|
}
|
||||||
|
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
|
||||||
|
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
|
||||||
|
// runs the module is on neither list, its removed want left as it is.
|
||||||
|
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
before map[string]delivery.Want
|
||||||
|
want map[string]delivery.Want
|
||||||
|
}{
|
||||||
|
{"no key: nothing written", nil, map[string]delivery.Want{}},
|
||||||
|
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
|
||||||
|
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
|
||||||
|
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
|
||||||
|
map[string]delivery.Want{}},
|
||||||
|
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
|
||||||
|
} {
|
||||||
|
out := Wants(m, n, c.before, nil)
|
||||||
|
if len(out.Wants) != len(c.want) {
|
||||||
|
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for node, w := range c.want {
|
||||||
|
if !delivery.SameWant(normal(out.Wants[node]), w) {
|
||||||
|
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
m.FP = ""
|
m.FP = ""
|
||||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
||||||
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
|
||||||
|
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
|
||||||
|
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
|
||||||
|
// other program: it serves no tool (ADR 0437 §4).
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
|
||||||
|
"mesh-delivery-builder/internal/build"
|
||||||
|
)
|
||||||
|
|
||||||
|
// queueAsker writes a queues value and waits for the builder's answer to it.
|
||||||
|
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||||
|
|
||||||
|
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||||
|
// the builder's read.
|
||||||
|
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
|
||||||
|
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||||
|
return []stdio.Tool{
|
||||||
|
{Name: build.StuckTool,
|
||||||
|
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||||
|
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||||
|
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||||
|
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||||
|
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||||
|
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||||
|
{Name: build.AskAgainTool,
|
||||||
|
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
|
||||||
|
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
|
||||||
|
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
|
||||||
|
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
|
||||||
|
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||||
|
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||||
|
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
|
||||||
|
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := askAgain(key, value); err != nil {
|
||||||
|
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
|
||||||
|
}
|
||||||
|
return map[string]any{"key": key, "asked": true,
|
||||||
|
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||||
|
}},
|
||||||
|
{Name: build.SeatQueueTool,
|
||||||
|
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||||
|
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||||
|
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
|
||||||
|
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
|
||||||
|
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||||
|
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
|
||||||
|
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
|
||||||
|
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
language := build.LanguageOf(str(args, "language"))
|
||||||
|
if language == "" {
|
||||||
|
language = build.LanguageOf(str(args, "seat"))
|
||||||
|
}
|
||||||
|
if language == "" {
|
||||||
|
language = delivery.AskGo
|
||||||
|
}
|
||||||
|
limit, _ := args["limit"].(float64)
|
||||||
|
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return queue(key, value)
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||||
|
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||||
|
return func(key string, value []byte) (json.RawMessage, error) {
|
||||||
|
answered := make(chan json.RawMessage, 1)
|
||||||
|
var rev uint64
|
||||||
|
var mu sync.Mutex
|
||||||
|
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||||
|
mu.Lock()
|
||||||
|
mine := rev
|
||||||
|
mu.Unlock()
|
||||||
|
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
|
||||||
|
select {
|
||||||
|
case answered <- c.Value:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
|
||||||
|
}
|
||||||
|
r, err := s.Put(key, json.RawMessage(value))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
rev = r
|
||||||
|
mu.Unlock()
|
||||||
|
// The watch may have heard the answer before the revision was known: read it once too.
|
||||||
|
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
|
||||||
|
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
|
||||||
|
return e.Value, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||||
|
defer cancel()
|
||||||
|
select {
|
||||||
|
case v := <-answered:
|
||||||
|
return v, nil
|
||||||
|
case <-wait.Done():
|
||||||
|
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func read(s stdio.KeptState) build.StateRead {
|
||||||
|
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
|
||||||
|
e, err := s.Get(key)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, time.Time{}, err
|
||||||
|
}
|
||||||
|
return e.Value, time.Time{}, nil
|
||||||
|
}, Written: func(key string) time.Time {
|
||||||
|
e, err := s.Get(key)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return written(s, key, e.Revision)
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// written is when the key's revision was written, from its history, or zero.
|
||||||
|
func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
||||||
|
h, err := s.History(key)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
for _, r := range h {
|
||||||
|
if r.Revision == revision {
|
||||||
|
return r.At
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
steps := map[string]build.StateRead{}
|
||||||
|
for _, bm := range build.StuckBuildModules {
|
||||||
|
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||||
|
}
|
||||||
|
reasks := stdio.State(build.ReAsksState)
|
||||||
|
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||||
|
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||||
|
put := func(key string, value []byte) error {
|
||||||
|
_, err := reasks.Put(key, json.RawMessage(value))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,73 +4,303 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-builder/internal/build"
|
||||||
)
|
)
|
||||||
|
|
||||||
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||||
// (issue 1910): read once, at the take-over of a key at start, never polled.
|
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
|
||||||
|
// be answered (issue 1945). Never polled.
|
||||||
type heldOnTheBus struct{ js jetstream.JetStream }
|
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||||
|
|
||||||
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||||
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||||
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
||||||
module, err := delivery.BuildModuleOf(a.Language)
|
seen, err := h.seen(ctx, a, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
pattern, err := delivery.AskStepWatch(a.ID)
|
return seen.Step || seen.OnStream, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
|
||||||
|
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
|
||||||
|
seen, err := h.seen(ctx, a, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return "", err
|
||||||
|
}
|
||||||
|
return seen.Why(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
|
||||||
|
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
|
||||||
|
const (
|
||||||
|
maxOnSubject = 5000
|
||||||
|
maxSteps = 200
|
||||||
|
)
|
||||||
|
|
||||||
|
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
|
||||||
|
// the ask is on its stream.
|
||||||
|
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
|
||||||
|
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
|
||||||
|
module, err := delivery.BuildModuleOf(a.Language)
|
||||||
|
if err != nil {
|
||||||
|
return seen, err
|
||||||
}
|
}
|
||||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||||
|
}
|
||||||
|
pattern, err := delivery.AskStepWatch(a.ID)
|
||||||
|
if err != nil {
|
||||||
|
return seen, err
|
||||||
}
|
}
|
||||||
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return seen, err
|
||||||
}
|
}
|
||||||
for range keys.Keys() {
|
for range keys.Keys() {
|
||||||
keys.Stop()
|
keys.Stop()
|
||||||
return true, nil // a node took it
|
seen.Step = true // a node took it
|
||||||
|
return seen, nil
|
||||||
|
}
|
||||||
|
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||||
|
seen.Stream = name
|
||||||
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
return seen, nil
|
||||||
}
|
}
|
||||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return seen, err
|
||||||
|
}
|
||||||
|
var ahead []build.Queued
|
||||||
|
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||||
|
if q.Ask == a.ID {
|
||||||
|
seen.OnStream, seen.PutAt = true, q.PutAt
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ahead = append(ahead, q)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
if err != nil || !seen.OnStream || !later {
|
||||||
|
return seen, err
|
||||||
|
}
|
||||||
|
taken, err := stepKeys(ctx, steps)
|
||||||
|
if err != nil {
|
||||||
|
return seen, err
|
||||||
|
}
|
||||||
|
for i := range ahead {
|
||||||
|
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||||
|
}
|
||||||
|
seen.Ahead = ahead
|
||||||
|
since := seen.At
|
||||||
|
if since.IsZero() {
|
||||||
|
since = seen.PutAt
|
||||||
|
}
|
||||||
|
for _, q := range ahead {
|
||||||
|
if q.At.Before(since) {
|
||||||
|
since = q.At
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
|
||||||
|
return seen, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stream is the build seat's stream of language, its name and the subject of kind on it.
|
||||||
|
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
|
||||||
|
subject, err := delivery.AskSubject(language, kind)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", "", err
|
||||||
}
|
}
|
||||||
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
||||||
// messages by sequence and nothing wider (issue 1910).
|
// messages by sequence and nothing wider (issue 1910).
|
||||||
name, err := delivery.AskStream(a.Language)
|
name, err := delivery.AskStream(language)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return nil, "", "", err
|
||||||
}
|
}
|
||||||
stream, err := h.js.Stream(ctx, name)
|
stream, err := h.js.Stream(ctx, name)
|
||||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
return stream, name, subject, err
|
||||||
return false, nil
|
}
|
||||||
}
|
|
||||||
if err != nil {
|
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
|
||||||
return false, err
|
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
|
||||||
}
|
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
|
||||||
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
|
// none consumed.
|
||||||
// instead, each a direct read that consumes nothing.
|
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
|
||||||
info, err := stream.Info(ctx)
|
for seq, n := uint64(1), 0; ; n++ {
|
||||||
if err != nil {
|
if n >= maxOnSubject {
|
||||||
return false, err
|
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
|
||||||
}
|
}
|
||||||
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
|
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
|
||||||
m, err := stream.GetMsg(ctx, seq)
|
|
||||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||||
continue // taken and acknowledged, or removed
|
return nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return err
|
||||||
}
|
}
|
||||||
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
|
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
|
||||||
return true, nil
|
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
|
||||||
|
q.Kind = kind
|
||||||
|
}
|
||||||
|
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
|
||||||
|
q.At = q.PutAt
|
||||||
|
}
|
||||||
|
if !each(q) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seq = m.Sequence + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
|
||||||
|
// ask id and is left out).
|
||||||
|
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
|
||||||
|
all, err := steps.ListKeys(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||||
|
return map[string][]string{}, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer all.Stop()
|
||||||
|
out := map[string][]string{}
|
||||||
|
for k := range all.Keys() {
|
||||||
|
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
|
||||||
|
out[id] = append(out[id], k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepState is the state of the first of keys a node wrote, "" for none.
|
||||||
|
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
|
||||||
|
for _, k := range keys {
|
||||||
|
e, err := steps.Get(ctx, k)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if s, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||||
|
return s.State
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
|
||||||
|
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
|
||||||
|
// (mesh-sdk buildstep).
|
||||||
|
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
|
||||||
|
var ids []string
|
||||||
|
for id := range taken {
|
||||||
|
if build.AskedAt(id).After(since) {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
|
||||||
|
read := 0
|
||||||
|
for _, id := range ids {
|
||||||
|
for _, k := range taken[id] {
|
||||||
|
if read >= maxSteps {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
read++
|
||||||
|
e, err := steps.Get(ctx, k)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s, err := delivery.DecodeStep(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
|
||||||
|
if !ok || !build.Proves(stepKind, kind) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
|
||||||
|
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
|
||||||
|
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
|
||||||
|
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||||
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var seq uint64
|
||||||
|
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||||
|
if q.Ask == a.ID {
|
||||||
|
seq = q.Seq
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}); err != nil || seq == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||||
|
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
|
||||||
|
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
|
||||||
|
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
|
||||||
|
q := build.SeatQueue{Language: language, Note: build.QueueNote}
|
||||||
|
module, err := delivery.BuildModuleOf(language)
|
||||||
|
if err != nil {
|
||||||
|
return q, err
|
||||||
|
}
|
||||||
|
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
|
||||||
|
q.Stream = name
|
||||||
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
q.Note = name + " does not exist"
|
||||||
|
return q, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return q, err
|
||||||
|
}
|
||||||
|
info, err := stream.Info(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return q, err
|
||||||
|
}
|
||||||
|
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
|
||||||
|
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||||
|
if err != nil {
|
||||||
|
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||||
|
}
|
||||||
|
taken, err := stepKeys(ctx, steps)
|
||||||
|
if err != nil {
|
||||||
|
return q, err
|
||||||
|
}
|
||||||
|
for _, kind := range delivery.AskKinds {
|
||||||
|
subject, _ := delivery.AskSubject(language, kind)
|
||||||
|
var ahead []build.Queued
|
||||||
|
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
|
||||||
|
ahead = append(ahead, m)
|
||||||
|
return len(ahead) < limit
|
||||||
|
}); err != nil {
|
||||||
|
return q, err
|
||||||
|
}
|
||||||
|
if len(ahead) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := range ahead {
|
||||||
|
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||||
|
}
|
||||||
|
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
|
||||||
|
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
|
||||||
|
}
|
||||||
|
return q, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ import (
|
|||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-builder/internal/build"
|
||||||
)
|
)
|
||||||
|
|
||||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||||
@@ -822,6 +824,34 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
|||||||
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
|
||||||
|
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
|
||||||
|
held := heldOnTheBus{b.js}
|
||||||
|
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||||
|
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||||
|
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
|
||||||
|
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
queued := func() []build.QueuedState {
|
||||||
|
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
|
||||||
|
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
|
||||||
|
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
|
||||||
|
}
|
||||||
|
return q.Kinds[0].Messages
|
||||||
|
}
|
||||||
|
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
|
||||||
|
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
|
||||||
|
}
|
||||||
|
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
|
||||||
|
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
|
||||||
|
}
|
||||||
|
if err := held.Drop(b.ctx, stale); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
|
||||||
|
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
|
||||||
|
}
|
||||||
b.stop()
|
b.stop()
|
||||||
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
||||||
b.fake.open()
|
b.fake.open()
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"maps"
|
"maps"
|
||||||
@@ -25,6 +24,8 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -124,29 +125,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||||
kv, err := js.KeyValue(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
|
||||||
}
|
|
||||||
return kv, nil
|
|
||||||
}
|
|
||||||
builds, err := bucket(delivery.BuilderBuildsBucket)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
modules, err := bucket(delivery.ModulesBucket)
|
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
manifests, err := bucket(delivery.ManifestsBucket)
|
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
candidates, err := bucket(delivery.CandidatesBucket)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
|
||||||
|
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
|
||||||
bound := defaultStepBound
|
bound := defaultStepBound
|
||||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||||
if bound, err = time.ParseDuration(v); err != nil {
|
if bound, err = time.ParseDuration(v); err != nil {
|
||||||
@@ -157,6 +149,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||||
asks := buildrun.JetStreamAsks{JS: js}
|
asks := buildrun.JetStreamAsks{JS: js}
|
||||||
|
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||||
|
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||||
|
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||||
|
// state hearing, which builder_stuck says (dead_letters).
|
||||||
|
dead := &buildrun.DeadLetters{}
|
||||||
|
hearingKV := process.OptionalBucket(ctx, js, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||||
|
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||||
|
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||||
|
defer hearing.Stop()
|
||||||
|
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||||
account, err := process.ReadAccount(path)
|
account, err := process.ReadAccount(path)
|
||||||
@@ -169,44 +171,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
defer calls.Close()
|
defer calls.Close()
|
||||||
b := &build.Builder{
|
b := &build.Builder{
|
||||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||||
Held: heldOnTheBus{js},
|
Held: heldOnTheBus{js},
|
||||||
|
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||||
|
Gone: heldOnTheBus{js},
|
||||||
|
CheckAfter: bound,
|
||||||
Toolchains: tools,
|
Toolchains: tools,
|
||||||
Manifests: manifestReader{manifests},
|
Manifests: manifestReader{manifests},
|
||||||
Logf: logger.Printf,
|
Logf: logger.Printf,
|
||||||
Stored: storedVia(calls.Call),
|
Stored: storedVia(calls.Call),
|
||||||
}
|
}
|
||||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||||
// revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said.
|
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||||
// The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a
|
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||||
// revision heard after the candidate that names it builds that candidate then.
|
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||||
trunks := &sourceHeads{}
|
errs := make(chan error, 6)
|
||||||
var sw jetstream.KeyWatcher
|
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
trunks.start()
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
b.Trunk = trunks.head
|
||||||
logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket)
|
b.Said = func(e build.AskedAgain) {
|
||||||
case err != nil:
|
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
logger.Printf("%v", err)
|
||||||
default:
|
|
||||||
w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err)
|
|
||||||
}
|
}
|
||||||
if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
b.Trunk = trunks.head
|
|
||||||
}
|
}
|
||||||
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||||
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||||
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||||
// is asked since R4g, ADR 0524 §7).
|
// is asked since R4g, ADR 0524 §7).
|
||||||
switch kv, err := js.KeyValue(ctx, instancesBucket); {
|
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
case kv == nil:
|
||||||
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
|
|
||||||
case err != nil:
|
|
||||||
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
|
|
||||||
default:
|
default:
|
||||||
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||||
b.Answered = says.Answered
|
b.Answered = says.Answered
|
||||||
@@ -286,23 +280,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
b.ForgetCandidate(e.Key())
|
b.ForgetCandidate(e.Key())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cw, err := settle(ctx, candidates, eachCandidate)
|
if candidates != nil {
|
||||||
if err != nil {
|
cw, err := settle(ctx, candidates, eachCandidate)
|
||||||
return err
|
if err != nil {
|
||||||
|
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
|
||||||
|
} else {
|
||||||
|
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||||
|
// every other input. The builder asks again by itself without it.
|
||||||
|
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||||
|
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||||
|
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||||
|
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||||
|
} else {
|
||||||
|
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||||
|
// builder alone may read the stream, so it answers there.
|
||||||
|
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||||
|
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||||
|
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||||
|
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||||
|
} else {
|
||||||
|
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
logger.Printf("started on %s", cred.Node)
|
logger.Printf("started on %s", cred.Node)
|
||||||
errs := make(chan error, 3)
|
|
||||||
go func() { errs <- follow(ctx, w, each) }()
|
go func() { errs <- follow(ctx, w, each) }()
|
||||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
|
||||||
if sw != nil {
|
|
||||||
go func() {
|
|
||||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
|
||||||
if trunks.put(e) {
|
|
||||||
b.TrunkMoved(ctx)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
b.Wait()
|
b.Wait()
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
|
||||||
|
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
|
||||||
|
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
|
||||||
|
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
|
||||||
|
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
|
||||||
|
// and opened again at the next event that needs it where there is one.
|
||||||
|
|
||||||
|
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
|
||||||
|
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
|
||||||
|
// and a candidate's preview waits, saying why.
|
||||||
|
type trunkSource struct {
|
||||||
|
ctx context.Context
|
||||||
|
js bucketOpener
|
||||||
|
heads *sourceHeads
|
||||||
|
moved func(context.Context)
|
||||||
|
errs chan<- error
|
||||||
|
logf func(string, ...any)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
opened bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
|
||||||
|
func (t *trunkSource) ensure() error {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.opened {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
|
||||||
|
}
|
||||||
|
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
|
||||||
|
}
|
||||||
|
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||||
|
}
|
||||||
|
t.opened = true
|
||||||
|
go func() {
|
||||||
|
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||||
|
if t.heads.put(e) && t.moved != nil {
|
||||||
|
t.moved(t.ctx)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// start opens the source at start; one that cannot be is said, and the builder goes on.
|
||||||
|
func (t *trunkSource) start() {
|
||||||
|
if err := t.ensure(); err != nil {
|
||||||
|
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
|
||||||
|
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||||
|
if err := t.ensure(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return t.heads.head(ctx, rev, repository)
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"mesh-delivery-builder/internal/build"
|
||||||
|
)
|
||||||
|
|
||||||
|
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
|
||||||
|
type queueReader interface {
|
||||||
|
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
|
||||||
|
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
|
||||||
|
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
|
||||||
|
// instance's) or deleted is left.
|
||||||
|
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||||
|
if e.Operation() != jetstream.KeyValuePut {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a, err := build.DecodeQueueAsk(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !a.Answered.IsZero() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
read, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||||
|
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
a.Error = err.Error()
|
||||||
|
}
|
||||||
|
a.Queue, a.Answered = &q, time.Now().UTC()
|
||||||
|
data, err := json.Marshal(a)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||||
|
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"mesh-delivery-builder/internal/build"
|
||||||
|
)
|
||||||
|
|
||||||
|
// askAgainer is the part of the builder a reasks value asks through.
|
||||||
|
type askAgainer interface {
|
||||||
|
AskAgain(key, why string) []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||||
|
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||||
|
// answered already (this builder's own write) or deleted is left.
|
||||||
|
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||||
|
if e.Operation() != jetstream.KeyValuePut {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r, err := build.DecodeReAsk(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Answer != "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
asked := b.AskAgain(e.Key(), r.Why)
|
||||||
|
if len(asked) == 0 {
|
||||||
|
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||||
|
} else {
|
||||||
|
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
|
||||||
|
}
|
||||||
|
r.Answered = time.Now().UTC()
|
||||||
|
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||||
|
data, err := json.Marshal(r)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||||
|
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||||
|
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||||
|
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
|
||||||
|
data, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
|
||||||
|
id := "asked-again-" + e.New
|
||||||
|
msg.Header.Set("x-event-id", id)
|
||||||
|
msg.Header.Set("x-source", cred.Module)
|
||||||
|
msg.Header.Set("x-node", cred.Node)
|
||||||
|
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||||
|
msg.Header.Set("content-type", "application/json")
|
||||||
|
msg.Data = data
|
||||||
|
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||||
|
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||||
w, err := settle(ctx, kv, each)
|
w, err := settle(ctx, kv, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||||
|
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
|
return errs, nil
|
||||||
}
|
}
|
||||||
if gate.Settled() {
|
if gate.Settled() {
|
||||||
changed(ctx)
|
changed(ctx)
|
||||||
|
|||||||
@@ -3,11 +3,19 @@ package main
|
|||||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
@@ -89,6 +97,61 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
if n := len(set.Entries("typescript")); n != 2 {
|
if n := len(set.Entries("typescript")); n != 2 {
|
||||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||||
|
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||||
|
// is an error naming the bucket and its grant.
|
||||||
|
var logs bytes.Buffer
|
||||||
|
logger := log.New(&logs, "", 0)
|
||||||
|
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||||
|
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||||
|
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||||
|
}
|
||||||
|
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||||
|
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||||
|
t.Errorf("the refusals were not said: %q", logs.String())
|
||||||
|
}
|
||||||
|
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||||
|
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||||
|
}
|
||||||
|
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||||
|
// head is told why and opens it again then.
|
||||||
|
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||||
|
src.start()
|
||||||
|
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||||
|
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||||
|
}
|
||||||
|
if src.opened {
|
||||||
|
t.Error("a refused source counts as opened, so it is never opened again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||||
|
type refusingBus struct {
|
||||||
|
refuseOpen bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||||
|
if r.refuseOpen {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
return refusedWatch{err: r.err}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type refusedWatch struct {
|
||||||
|
jetstream.KeyValue
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||||
|
return nil, r.err
|
||||||
}
|
}
|
||||||
|
|
||||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||||
|
|||||||
@@ -4,11 +4,13 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -62,11 +64,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
|||||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||||
return nil
|
return nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||||
|
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
w.watched[module] = true
|
w.watched[module] = true
|
||||||
go func() {
|
go func() {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026134035-21252194d530
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026134035-21252194d530 h1:NwCb+gOh0UYqlyG6hG2/jffE/Gd0c7YWd4iJlaCB0lU=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026134035-21252194d530/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -0,0 +1,344 @@
|
|||||||
|
package build
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||||
|
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
|
||||||
|
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
|
||||||
|
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||||
|
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||||
|
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||||
|
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||||
|
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||||
|
|
||||||
|
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||||
|
const DefaultAskings = 3
|
||||||
|
|
||||||
|
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
|
||||||
|
const AskedAgainEvent = "build.asked-again"
|
||||||
|
|
||||||
|
// AskedAgain is the body of AskedAgainEvent.
|
||||||
|
type AskedAgain struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
FP string `json:"fp"`
|
||||||
|
BuildModule string `json:"build_module"`
|
||||||
|
Language string `json:"language"`
|
||||||
|
Old string `json:"old"`
|
||||||
|
New string `json:"new"`
|
||||||
|
Asking int `json:"asking"`
|
||||||
|
Askings int `json:"askings"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
|
||||||
|
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
|
||||||
|
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
|
||||||
|
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
|
||||||
|
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
|
||||||
|
// five hours, each counted as waiting, so the ask was never asked again).
|
||||||
|
type AskSeen struct {
|
||||||
|
Stream string
|
||||||
|
// Kind is the ask's kind, At when it was asked (its id's time).
|
||||||
|
Kind string
|
||||||
|
At time.Time
|
||||||
|
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
|
||||||
|
Step bool
|
||||||
|
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
|
||||||
|
OnStream bool
|
||||||
|
PutAt time.Time
|
||||||
|
// Ahead are the messages of its subject before it on the stream.
|
||||||
|
Ahead []Queued
|
||||||
|
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
|
||||||
|
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
|
||||||
|
Last *Taken
|
||||||
|
}
|
||||||
|
|
||||||
|
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
|
||||||
|
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
|
||||||
|
type Queued struct {
|
||||||
|
Seq uint64 `json:"seq"`
|
||||||
|
Ask string `json:"ask"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
At time.Time `json:"asked"`
|
||||||
|
PutAt time.Time `json:"put"`
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
|
||||||
|
type Taken struct {
|
||||||
|
Ask string `json:"ask"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
At time.Time `json:"asked"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
|
||||||
|
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
|
||||||
|
// its worker had none of waiting's kind left to give.
|
||||||
|
func Proves(taken, waiting string) bool {
|
||||||
|
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
|
||||||
|
return t >= 0 && w >= 0 && t >= w
|
||||||
|
}
|
||||||
|
|
||||||
|
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
|
||||||
|
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
|
||||||
|
// its age, and is not counted.
|
||||||
|
func (s AskSeen) Waits(q Queued) bool {
|
||||||
|
return q.Step == "" && !s.passed(q.At)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s AskSeen) passed(at time.Time) bool {
|
||||||
|
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Waiting is how many messages ahead still wait to be delivered (Waits).
|
||||||
|
func (s AskSeen) Waiting() int {
|
||||||
|
n := 0
|
||||||
|
for _, q := range s.Ahead {
|
||||||
|
if s.Waits(q) {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
|
||||||
|
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
|
||||||
|
// ADR 0525 §9).
|
||||||
|
func (s AskSeen) Why() string {
|
||||||
|
switch {
|
||||||
|
case s.Step:
|
||||||
|
return ""
|
||||||
|
case !s.OnStream:
|
||||||
|
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
|
||||||
|
case s.Waiting() > 0 || !s.passed(s.at()):
|
||||||
|
return ""
|
||||||
|
default:
|
||||||
|
behind := ""
|
||||||
|
if n := len(s.Ahead); n > 0 {
|
||||||
|
behind = fmt.Sprintf(" behind %d given up on or taken", n)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
|
||||||
|
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s AskSeen) at() time.Time {
|
||||||
|
if s.At.IsZero() {
|
||||||
|
return s.PutAt
|
||||||
|
}
|
||||||
|
return s.At
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
|
||||||
|
func AskedAt(id string) time.Time {
|
||||||
|
if len(id) <= 6 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||||
|
if err != nil || ms <= 0 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return time.UnixMilli(ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
|
||||||
|
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
|
||||||
|
type Dropper interface {
|
||||||
|
Drop(ctx context.Context, a delivery.StepAsk) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
|
||||||
|
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||||
|
d, ok := b.Gone.(Dropper)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prior := ask
|
||||||
|
prior.ID = old
|
||||||
|
if err := d.Drop(ctx, prior); err != nil {
|
||||||
|
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Builder) askings() int {
|
||||||
|
if b.Askings > 0 {
|
||||||
|
return b.Askings
|
||||||
|
}
|
||||||
|
return DefaultAskings
|
||||||
|
}
|
||||||
|
|
||||||
|
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
|
||||||
|
|
||||||
|
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
|
||||||
|
func askingOf(step, buildModule string) int {
|
||||||
|
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
|
||||||
|
if m[1] == buildModule {
|
||||||
|
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
|
||||||
|
// languages whose lane waits, none when no lane of this builder waits on the key.
|
||||||
|
func (b *Builder) AskAgain(key, why string) []string {
|
||||||
|
b.init()
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
var asked []string
|
||||||
|
for language, c := range b.waiting[key] {
|
||||||
|
select {
|
||||||
|
case c <- why:
|
||||||
|
default: // one is already asked
|
||||||
|
}
|
||||||
|
asked = append(asked, language)
|
||||||
|
}
|
||||||
|
return asked
|
||||||
|
}
|
||||||
|
|
||||||
|
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
|
||||||
|
func (b *Builder) Waiting() map[string][]string {
|
||||||
|
b.init()
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
out := map[string][]string{}
|
||||||
|
for key, ls := range b.waiting {
|
||||||
|
for l := range ls {
|
||||||
|
out[key] = append(out[key], l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Builder) listen(key, language string) chan string {
|
||||||
|
b.init()
|
||||||
|
c := make(chan string, 1)
|
||||||
|
b.mu.Lock()
|
||||||
|
if b.waiting[key] == nil {
|
||||||
|
b.waiting[key] = map[string]chan string{}
|
||||||
|
}
|
||||||
|
b.waiting[key][language] = c
|
||||||
|
b.mu.Unlock()
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Builder) unlisten(key, language string, c chan string) {
|
||||||
|
b.mu.Lock()
|
||||||
|
if b.waiting[key][language] == c {
|
||||||
|
delete(b.waiting[key], language)
|
||||||
|
if len(b.waiting[key]) == 0 {
|
||||||
|
delete(b.waiting, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
errCheck = errors.New("the check bound passed")
|
||||||
|
errAgain = errors.New("asked again")
|
||||||
|
)
|
||||||
|
|
||||||
|
type byHand struct{ why string }
|
||||||
|
|
||||||
|
func (h byHand) Error() string { return "asked again by hand: " + h.why }
|
||||||
|
|
||||||
|
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
|
||||||
|
// asked it again (AskAgain); it answers the step and the ask that has it.
|
||||||
|
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
|
||||||
|
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||||
|
again := b.listen(key, ask.Language)
|
||||||
|
defer b.unlisten(key, ask.Language, again)
|
||||||
|
asking := askingOf(asked.step, l.module)
|
||||||
|
for {
|
||||||
|
step, why, err := b.waitOnce(ctx, ask, again)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, errCheck):
|
||||||
|
continue
|
||||||
|
case errors.Is(err, errAgain):
|
||||||
|
if asking >= b.askings() {
|
||||||
|
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||||
|
}
|
||||||
|
asking++
|
||||||
|
old := ask.ID
|
||||||
|
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||||
|
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
|
||||||
|
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
|
||||||
|
}
|
||||||
|
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||||
|
return step, ask, err
|
||||||
|
}
|
||||||
|
b.drop(ctx, ask, old)
|
||||||
|
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
|
||||||
|
if b.Said != nil {
|
||||||
|
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
|
||||||
|
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return step, ask, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
|
||||||
|
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
|
||||||
|
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
|
||||||
|
hand, cancel := context.WithCancelCause(ctx)
|
||||||
|
defer cancel(nil)
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case why := <-again:
|
||||||
|
cancel(byHand{why})
|
||||||
|
case <-hand.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
wait := hand
|
||||||
|
if b.Gone != nil && b.CheckAfter > 0 {
|
||||||
|
var stop context.CancelFunc
|
||||||
|
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
|
||||||
|
defer stop()
|
||||||
|
}
|
||||||
|
step, err := b.Asks.Wait(wait, ask)
|
||||||
|
if err == nil && buildrun.IsLost(step) {
|
||||||
|
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||||
|
// words of why, not at the check bound.
|
||||||
|
return step, step.Reason, errAgain
|
||||||
|
}
|
||||||
|
if err == nil || ctx.Err() != nil {
|
||||||
|
return step, "", err
|
||||||
|
}
|
||||||
|
var h byHand
|
||||||
|
if errors.As(context.Cause(hand), &h) {
|
||||||
|
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
|
||||||
|
}
|
||||||
|
if errors.Is(context.Cause(wait), errCheck) {
|
||||||
|
why, gerr := b.Gone.Gone(ctx, ask)
|
||||||
|
if gerr != nil {
|
||||||
|
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
|
||||||
|
return step, "", errCheck
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
|
return step, "", errCheck
|
||||||
|
}
|
||||||
|
return step, why, errAgain
|
||||||
|
}
|
||||||
|
return step, "", err
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package build
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
|
||||||
|
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
|
||||||
|
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
|
||||||
|
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
|
||||||
|
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
|
||||||
|
// there, as builder_ask_again does through reasks.
|
||||||
|
const (
|
||||||
|
SeatQueueTool = "builder_seat_queue"
|
||||||
|
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
|
||||||
|
QueuesState = "queues"
|
||||||
|
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
|
||||||
|
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
|
||||||
|
DefaultQueueLimit = 50
|
||||||
|
MaxQueueLimit = 500
|
||||||
|
)
|
||||||
|
|
||||||
|
// QueueNote says what the read cannot show.
|
||||||
|
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
|
||||||
|
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
|
||||||
|
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
|
||||||
|
|
||||||
|
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
|
||||||
|
type QueueAsk struct {
|
||||||
|
V int `json:"v"`
|
||||||
|
Limit int `json:"limit"`
|
||||||
|
By string `json:"by,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Answered time.Time `json:"answered,omitzero"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Queue *SeatQueue `json:"queue,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SeatQueue is what the builder read of one build seat's stream.
|
||||||
|
type SeatQueue struct {
|
||||||
|
Language string `json:"language"`
|
||||||
|
Stream string `json:"stream"`
|
||||||
|
Messages uint64 `json:"messages"`
|
||||||
|
FirstSeq uint64 `json:"first_seq"`
|
||||||
|
LastSeq uint64 `json:"last_seq"`
|
||||||
|
Kinds []KindQueue `json:"kinds"`
|
||||||
|
Note string `json:"note"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
|
||||||
|
// last a build node took of the kind (or of one pulled after it), and each message.
|
||||||
|
type KindQueue struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Read int `json:"read"`
|
||||||
|
Waiting int `json:"waiting"`
|
||||||
|
GivenUp int `json:"given_up_or_taken"`
|
||||||
|
Last *Taken `json:"last_taken,omitempty"`
|
||||||
|
Messages []QueuedState `json:"messages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueuedState is one message as the tool shows it.
|
||||||
|
type QueuedState struct {
|
||||||
|
Queued
|
||||||
|
AgeSeconds int64 `json:"age_s"`
|
||||||
|
Waits bool `json:"waits"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewKindQueue is seen's messages ahead as one kind's queue at now.
|
||||||
|
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
|
||||||
|
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
|
||||||
|
for _, q := range seen.Ahead {
|
||||||
|
w := seen.Waits(q)
|
||||||
|
if w {
|
||||||
|
k.Waiting++
|
||||||
|
} else {
|
||||||
|
k.GivenUp++
|
||||||
|
}
|
||||||
|
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
|
||||||
|
}
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
|
||||||
|
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
|
||||||
|
if _, err := delivery.AskStream(language); err != nil {
|
||||||
|
return "", nil, fmt.Errorf("language: %w", err)
|
||||||
|
}
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = DefaultQueueLimit
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
|
||||||
|
return language, data, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeQueueAsk reads a queues value.
|
||||||
|
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
|
||||||
|
var a QueueAsk
|
||||||
|
if err := json.Unmarshal(data, &a); err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
|
if a.V != 1 {
|
||||||
|
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
|
||||||
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
|
||||||
|
func LanguageOf(arg string) string {
|
||||||
|
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
|
||||||
|
}
|
||||||
@@ -56,6 +56,18 @@ type Builder struct {
|
|||||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||||
|
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||||
|
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||||
|
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||||
|
Gone interface {
|
||||||
|
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
|
||||||
|
}
|
||||||
|
CheckAfter time.Duration
|
||||||
|
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||||
|
// with why; DefaultAskings when 0.
|
||||||
|
Askings int
|
||||||
|
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||||
|
Said func(AskedAgain)
|
||||||
|
|
||||||
once sync.Once
|
once sync.Once
|
||||||
sem chan struct{}
|
sem chan struct{}
|
||||||
@@ -75,6 +87,8 @@ type Builder struct {
|
|||||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||||
held map[string]bool
|
held map[string]bool
|
||||||
|
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||||
|
waiting map[string]map[string]chan string
|
||||||
}
|
}
|
||||||
|
|
||||||
type planned struct {
|
type planned struct {
|
||||||
@@ -91,7 +105,7 @@ func (b *Builder) init() {
|
|||||||
b.sem = make(chan struct{}, limit)
|
b.sem = make(chan struct{}, limit)
|
||||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||||
b.published, b.held = map[string]bool{}, map[string]bool{}
|
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -673,12 +687,14 @@ type askNotes struct {
|
|||||||
b *Builder
|
b *Builder
|
||||||
cl *Claim
|
cl *Claim
|
||||||
resumed map[string]string
|
resumed map[string]string
|
||||||
|
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
|
||||||
|
step string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Builder) notes(cl *Claim) *askNotes {
|
func (b *Builder) notes(cl *Claim) *askNotes {
|
||||||
n := &askNotes{b: b, cl: cl}
|
n := &askNotes{b: b, cl: cl}
|
||||||
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
||||||
n.resumed = maps.Clone(cl.Build.Asks)
|
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
|
||||||
}
|
}
|
||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
@@ -776,7 +792,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
step, err := b.Asks.Wait(ctx, ask)
|
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.err = fmt.Errorf("%s: %w", l.module, err)
|
l.err = fmt.Errorf("%s: %w", l.module, err)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -616,27 +618,304 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
||||||
// again; the key is done when a node takes it.
|
// drop from their stream (Dropper).
|
||||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
type goneAsks struct {
|
||||||
r := newRig(t, goBundle)
|
mu sync.Mutex
|
||||||
r.asks.Answer = nil
|
n int
|
||||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
why func(a delivery.StepAsk) string
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
dropped []string
|
||||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
}
|
||||||
|
|
||||||
|
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
g.dropped = append(g.dropped, a.ID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.n++
|
||||||
|
g.mu.Unlock()
|
||||||
|
return g.why(a), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *goneAsks) asked() int {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
return g.n
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForAsks waits until n asks were put.
|
||||||
|
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
|
||||||
|
t.Helper()
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
|
||||||
if time.Now().After(deadline) {
|
if time.Now().After(deadline) {
|
||||||
t.Fatal("no ask was put")
|
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
time.Sleep(100 * time.Millisecond)
|
return r.asks.Asks()
|
||||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
}
|
||||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
|
||||||
}
|
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
|
||||||
r.b.Wait()
|
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
|
||||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
t.Run("waits in its stream: never asked again", func(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
r.asks.Answer = nil
|
||||||
|
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||||
|
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
|
||||||
|
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||||
|
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||||
|
waitForAsks(t, r, 1)
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||||
|
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||||
|
}
|
||||||
|
if gone.asked() == 0 {
|
||||||
|
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
|
||||||
|
}
|
||||||
|
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||||
|
r.b.Wait()
|
||||||
|
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||||
|
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
askings int
|
||||||
|
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
|
||||||
|
step string
|
||||||
|
byHand bool
|
||||||
|
always bool // Gone says every ask cannot be answered, not only the first
|
||||||
|
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||||
|
puts int
|
||||||
|
done bool
|
||||||
|
asking int // the asking the event says, 0 when none is said
|
||||||
|
}{
|
||||||
|
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||||
|
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||||
|
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||||
|
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||||
|
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) { // 1945
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
r.asks.Answer = nil
|
||||||
|
r.b.Askings = c.askings
|
||||||
|
var said []AskedAgain
|
||||||
|
var mu sync.Mutex
|
||||||
|
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
||||||
|
first := ""
|
||||||
|
if c.step != "" {
|
||||||
|
first = "mvl0frv18d1960"
|
||||||
|
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||||
|
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
|
||||||
|
r.st.vals[r.key] = data
|
||||||
|
}
|
||||||
|
gone := &goneAsks{}
|
||||||
|
dead := &buildrun.DeadLetters{}
|
||||||
|
if c.dead {
|
||||||
|
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||||
|
}
|
||||||
|
if !c.byHand && !c.dead {
|
||||||
|
gone.why = func(a delivery.StepAsk) string {
|
||||||
|
if c.always || a.ID == first || first == "" {
|
||||||
|
return "it waits first in line while a node took a later ask"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||||
|
}
|
||||||
|
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||||
|
if c.dead {
|
||||||
|
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
|
||||||
|
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
|
||||||
|
}
|
||||||
|
if c.byHand {
|
||||||
|
waitForAsks(t, r, 1)
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("no lane of the key waits")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.done {
|
||||||
|
asks := waitForAsks(t, r, c.puts)
|
||||||
|
if first == "" && !c.byHand {
|
||||||
|
first = asks[0].ID
|
||||||
|
}
|
||||||
|
last := asks[len(asks)-1]
|
||||||
|
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
|
||||||
|
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
|
||||||
|
}
|
||||||
|
r.asks.End(r.answer(last))
|
||||||
|
}
|
||||||
|
r.b.Wait()
|
||||||
|
v := valueOf(t, r.st, r.key)
|
||||||
|
if n := len(r.asks.Asks()); n != c.puts {
|
||||||
|
t.Errorf("%d asks put, want %d", n, c.puts)
|
||||||
|
}
|
||||||
|
if c.done && v.State != delivery.BuildDone {
|
||||||
|
t.Errorf("the key = %+v, want done from the ask asked again", v)
|
||||||
|
}
|
||||||
|
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
|
||||||
|
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||||
|
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||||
|
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||||
|
}
|
||||||
|
gone.mu.Unlock()
|
||||||
|
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||||
|
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||||
|
}
|
||||||
|
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
|
||||||
|
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||||
|
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||||
|
taken := func(kind string, at time.Time) *Taken {
|
||||||
|
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
|
||||||
|
}
|
||||||
|
later := taken(delivery.AskDelivery, put.Add(time.Minute))
|
||||||
|
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
|
||||||
|
ahead := func(n int, step string) []Queued {
|
||||||
|
var qs []Queued
|
||||||
|
for i := range n {
|
||||||
|
at := put.Add(-time.Duration(n-i) * time.Minute)
|
||||||
|
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
|
||||||
|
}
|
||||||
|
return qs
|
||||||
|
}
|
||||||
|
seen := func(qs []Queued, last *Taken) AskSeen {
|
||||||
|
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
seen AskSeen
|
||||||
|
want string // a part of the why, "" when it can be answered
|
||||||
|
waiting int
|
||||||
|
}{
|
||||||
|
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
|
||||||
|
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
|
||||||
|
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
|
||||||
|
{"first in line, nothing taken since", seen(nil, nil), "", 0},
|
||||||
|
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||||
|
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
|
||||||
|
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||||
|
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
|
||||||
|
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
|
||||||
|
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
|
||||||
|
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
|
||||||
|
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
|
||||||
|
} {
|
||||||
|
got := c.seen.Why()
|
||||||
|
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
|
||||||
|
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
|
||||||
|
t.Errorf("AskedAt does not read an ask id's time")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
|
||||||
|
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||||
|
state := func(vals map[string]string) StateRead {
|
||||||
|
return StateRead{Keys: func() ([]string, error) {
|
||||||
|
var ks []string
|
||||||
|
for k := range vals {
|
||||||
|
ks = append(ks, k)
|
||||||
|
}
|
||||||
|
return ks, nil
|
||||||
|
}, Get: func(k string) (json.RawMessage, time.Time, error) {
|
||||||
|
if v, ok := vals[k]; ok {
|
||||||
|
return json.RawMessage(v), at, nil
|
||||||
|
}
|
||||||
|
return nil, time.Time{}, nil
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||||
|
done, _ := delivery.BuilderBuildKey("d", fp1)
|
||||||
|
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||||
|
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
|
||||||
|
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
|
||||||
|
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||||
|
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||||
|
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||||
|
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||||
|
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
|
||||||
|
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||||
|
out, err := r.Stuck("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
|
||||||
|
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
|
||||||
|
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
|
||||||
|
}
|
||||||
|
stuck := out["stuck"].([]Stuck)
|
||||||
|
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||||
|
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||||
|
}
|
||||||
|
a := stuck[0].Asks
|
||||||
|
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
|
||||||
|
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
|
||||||
|
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
|
||||||
|
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
|
||||||
|
// only for the rows kept.
|
||||||
|
var gets, written, inFlight, maxFlight atomic.Int64
|
||||||
|
vals := map[string]string{}
|
||||||
|
for i := 0; i < 3000; i++ {
|
||||||
|
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
|
||||||
|
vals[k] = string(finished)
|
||||||
|
if (i/100)%10 == 0 {
|
||||||
|
vals[k] = string(building)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slow := state(vals)
|
||||||
|
get := slow.Get
|
||||||
|
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
|
||||||
|
gets.Add(1)
|
||||||
|
n := inFlight.Add(1)
|
||||||
|
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
inFlight.Add(-1)
|
||||||
|
raw, _, err := get(k)
|
||||||
|
return raw, time.Time{}, err
|
||||||
|
}
|
||||||
|
slow.Written = func(string) time.Time { written.Add(1); return at }
|
||||||
|
for _, row := range []struct {
|
||||||
|
module string
|
||||||
|
stuck int
|
||||||
|
gets int64
|
||||||
|
cut bool
|
||||||
|
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
|
||||||
|
gets.Store(0)
|
||||||
|
written.Store(0)
|
||||||
|
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, cut := out["cut"]
|
||||||
|
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
|
||||||
|
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if maxFlight.Load() < 2 {
|
||||||
|
t.Errorf("the values were read one at a time")
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldAsks says an ask is held when its id is among held.
|
// heldAsks says an ask is held when its id is among held.
|
||||||
|
|||||||
@@ -0,0 +1,307 @@
|
|||||||
|
package build
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
||||||
|
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
|
||||||
|
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
|
||||||
|
// builder's state reasks, which the builder watches and answers there.
|
||||||
|
const (
|
||||||
|
StuckTool = "builder_stuck"
|
||||||
|
AskAgainTool = "builder_ask_again"
|
||||||
|
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||||
|
ReAsksState = "reasks"
|
||||||
|
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
|
||||||
|
// read it, its answer.
|
||||||
|
type ReAsk struct {
|
||||||
|
V int `json:"v"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
By string `json:"by,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Answer string `json:"answer,omitempty"`
|
||||||
|
Answered time.Time `json:"answered,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeReAsk reads a reasks value.
|
||||||
|
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||||
|
var r ReAsk
|
||||||
|
if err := json.Unmarshal(data, &r); err != nil {
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
if r.V != 1 {
|
||||||
|
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
|
||||||
|
}
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
|
||||||
|
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||||
|
key, err := delivery.BuilderBuildKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(why) == "" {
|
||||||
|
return "", nil, errors.New("say why the build is asked again (why)")
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||||
|
return key, data, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||||
|
type StateRead struct {
|
||||||
|
Keys func() ([]string, error)
|
||||||
|
Get func(key string) (json.RawMessage, time.Time, error)
|
||||||
|
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
|
||||||
|
// only for a row they keep, and only when Get did not say.
|
||||||
|
Written func(key string) time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// StuckAsk is one ask a key waits on.
|
||||||
|
type StuckAsk struct {
|
||||||
|
Language string `json:"language"`
|
||||||
|
BuildModule string `json:"build_module"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Asking int `json:"asking"`
|
||||||
|
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
|
||||||
|
// state on its node since its start.
|
||||||
|
Asked time.Time `json:"asked,omitzero"`
|
||||||
|
Step string `json:"step"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stuck is one build key that waits on an ask.
|
||||||
|
type Stuck struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
FP string `json:"fp"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Step string `json:"step"`
|
||||||
|
By string `json:"by"`
|
||||||
|
Since time.Time `json:"since,omitzero"`
|
||||||
|
Age string `json:"age,omitempty"`
|
||||||
|
Asks []StuckAsk `json:"asks"`
|
||||||
|
ReAsk *ReAsk `json:"reask,omitempty"`
|
||||||
|
Unasked bool `json:"unasked,omitempty"`
|
||||||
|
NoneStep int `json:"asks_with_no_step"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StuckReader reads what builder_stuck answers.
|
||||||
|
type StuckReader struct {
|
||||||
|
Builds StateRead
|
||||||
|
ReAsks StateRead
|
||||||
|
// Steps are the build modules' steps states, by build module.
|
||||||
|
Steps map[string]StateRead
|
||||||
|
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
||||||
|
Hearing StateRead
|
||||||
|
Now func() time.Time
|
||||||
|
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||||
|
Logf func(format string, args ...any)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
|
||||||
|
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
|
||||||
|
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
|
||||||
|
const (
|
||||||
|
ReadWorkers = 16
|
||||||
|
MaxReads = 1000
|
||||||
|
)
|
||||||
|
|
||||||
|
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
|
||||||
|
func parallel(n int, read func(i int) error) error {
|
||||||
|
var (
|
||||||
|
next atomic.Int64
|
||||||
|
failed atomic.Bool
|
||||||
|
once sync.Once
|
||||||
|
first error
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for !failed.Load() {
|
||||||
|
i := int(next.Add(1) - 1)
|
||||||
|
if i >= n {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := read(i); err != nil {
|
||||||
|
once.Do(func() { first = err })
|
||||||
|
failed.Store(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||||
|
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
||||||
|
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
||||||
|
func (r StuckReader) deadLetters() map[string]any {
|
||||||
|
if r.Hearing.Get == nil {
|
||||||
|
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
||||||
|
}
|
||||||
|
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
||||||
|
if err != nil {
|
||||||
|
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
||||||
|
}
|
||||||
|
a := buildrun.HearingHealth(raw)
|
||||||
|
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||||
|
started := time.Now()
|
||||||
|
reads := 0
|
||||||
|
defer func() {
|
||||||
|
if r.Logf != nil {
|
||||||
|
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
keys, err := r.Builds.Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
now := time.Now
|
||||||
|
if r.Now != nil {
|
||||||
|
now = r.Now
|
||||||
|
}
|
||||||
|
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
|
||||||
|
stepKeys := map[string]map[string]string{}
|
||||||
|
for bm, s := range r.Steps {
|
||||||
|
ks, err := s.Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
|
||||||
|
}
|
||||||
|
byID := make(map[string]string, len(ks))
|
||||||
|
for _, k := range ks {
|
||||||
|
if _, id, err := delivery.ParseStepKey(k); err == nil {
|
||||||
|
byID[id] = k
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stepKeys[bm] = byID
|
||||||
|
}
|
||||||
|
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
|
||||||
|
// parallel, at most MaxReads of them.
|
||||||
|
type parsed struct{ key, mod, fp string }
|
||||||
|
var matching []parsed
|
||||||
|
for _, key := range keys {
|
||||||
|
mod, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||||
|
if err != nil || (module != "" && mod != module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
matching = append(matching, parsed{key, mod, fp})
|
||||||
|
}
|
||||||
|
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
|
||||||
|
cut := len(matching) > MaxReads
|
||||||
|
total := len(matching)
|
||||||
|
if cut {
|
||||||
|
matching = matching[:MaxReads]
|
||||||
|
}
|
||||||
|
values := make([]json.RawMessage, len(matching))
|
||||||
|
ats := make([]time.Time, len(matching))
|
||||||
|
if err := parallel(len(matching), func(i int) error {
|
||||||
|
raw, at, err := r.Builds.Get(matching[i].key)
|
||||||
|
if err == nil {
|
||||||
|
values[i], ats[i] = raw, at
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
reads += len(matching)
|
||||||
|
out := []Stuck{}
|
||||||
|
for i, m := range matching {
|
||||||
|
key, mod, fp := m.key, m.mod, m.fp
|
||||||
|
raw, at := values[i], ats[i]
|
||||||
|
if raw == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v, err := delivery.DecodeBuilderBuild(raw)
|
||||||
|
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if at.IsZero() && r.Builds.Written != nil {
|
||||||
|
at = r.Builds.Written(key)
|
||||||
|
}
|
||||||
|
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
|
||||||
|
if !at.IsZero() {
|
||||||
|
s.Age = now().Sub(at).Round(time.Second).String()
|
||||||
|
}
|
||||||
|
for language, id := range v.Asks {
|
||||||
|
bm, _ := delivery.BuildModuleOf(language)
|
||||||
|
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
|
||||||
|
if t, ok := idTime(id); ok {
|
||||||
|
a.Asked = t
|
||||||
|
}
|
||||||
|
if st, ok := r.Steps[bm]; ok {
|
||||||
|
if k, ok := stepKeys[bm][id]; ok {
|
||||||
|
reads++
|
||||||
|
if raw, _, err := st.Get(k); err == nil && raw != nil {
|
||||||
|
if step, err := delivery.DecodeStep(raw); err == nil {
|
||||||
|
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.Step == "none" {
|
||||||
|
s.NoneStep++
|
||||||
|
}
|
||||||
|
s.Asks = append(s.Asks, a)
|
||||||
|
}
|
||||||
|
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
|
||||||
|
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
|
||||||
|
if r.ReAsks.Get != nil {
|
||||||
|
reads++
|
||||||
|
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
|
||||||
|
if ra, err := DecodeReAsk(raw); err == nil {
|
||||||
|
s.ReAsk = &ra
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if !out[i].Since.Equal(out[j].Since) {
|
||||||
|
return out[i].Since.Before(out[j].Since)
|
||||||
|
}
|
||||||
|
return out[i].Key < out[j].Key
|
||||||
|
})
|
||||||
|
answer := map[string]any{"stuck": out, "count": len(out),
|
||||||
|
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||||
|
answer["dead_letters"] = r.deadLetters()
|
||||||
|
if cut {
|
||||||
|
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StuckBuildModules are the build modules whose steps builder_stuck reads.
|
||||||
|
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
|
||||||
|
|
||||||
|
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
|
||||||
|
func idTime(id string) (time.Time, bool) {
|
||||||
|
if len(id) <= 6 {
|
||||||
|
return time.Time{}, false
|
||||||
|
}
|
||||||
|
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||||
|
if err != nil || ms <= 0 {
|
||||||
|
return time.Time{}, false
|
||||||
|
}
|
||||||
|
return time.UnixMilli(ms).UTC(), true
|
||||||
|
}
|
||||||
@@ -23,18 +23,21 @@ type state struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type artifact struct {
|
type artifact struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Language string `json:"language"`
|
Language string `json:"language"`
|
||||||
System string `json:"system"`
|
System string `json:"system"`
|
||||||
From string `json:"from"`
|
From string `json:"from"`
|
||||||
Binary string `json:"binary"`
|
Binary string `json:"binary"`
|
||||||
|
Loads []string `json:"loads"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type manifest struct {
|
type manifest struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
Slug string `json:"slug"`
|
Slug string `json:"slug"`
|
||||||
|
Tools []string `json:"tools"`
|
||||||
|
Emits []string `json:"emits"`
|
||||||
Consumes []string `json:"consumes"`
|
Consumes []string `json:"consumes"`
|
||||||
State []state `json:"state"`
|
State []state `json:"state"`
|
||||||
Reads []string `json:"reads"`
|
Reads []string `json:"reads"`
|
||||||
@@ -84,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
|||||||
if m.Module != "mesh-delivery-builder" {
|
if m.Module != "mesh-delivery-builder" {
|
||||||
t.Errorf("module = %q", m.Module)
|
t.Errorf("module = %q", m.Module)
|
||||||
}
|
}
|
||||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||||
}
|
}
|
||||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||||
@@ -112,7 +115,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
|||||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
|
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||||
|
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||||
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||||
|
}
|
||||||
|
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||||
|
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||||
if _, ok := top[k]; ok {
|
if _, ok := top[k]; ok {
|
||||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||||
}
|
}
|
||||||
@@ -127,7 +137,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
|||||||
procs = append(procs, r)
|
procs = append(procs, r)
|
||||||
}
|
}
|
||||||
if r["type"] == "tool" || r["type"] == "seat" {
|
if r["type"] == "tool" || r["type"] == "seat" {
|
||||||
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
|
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(procs) != 1 {
|
if len(procs) != 1 {
|
||||||
@@ -149,8 +159,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
|||||||
if _, bus := p["bus"]; bus {
|
if _, bus := p["bus"]; bus {
|
||||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||||
}
|
}
|
||||||
if len(m.Build.Artifacts) != 1 {
|
if len(m.Build.Artifacts) != 2 {
|
||||||
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
|
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||||
|
}
|
||||||
|
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||||
|
t.Errorf("tools artifact = %+v", a) // 1945
|
||||||
|
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||||
|
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||||
}
|
}
|
||||||
a := m.Build.Artifacts[0]
|
a := m.Build.Artifacts[0]
|
||||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||||
|
|||||||
@@ -2,6 +2,14 @@
|
|||||||
"module": "mesh-delivery-builder",
|
"module": "mesh-delivery-builder",
|
||||||
"version": "1",
|
"version": "1",
|
||||||
"slug": "builder",
|
"slug": "builder",
|
||||||
|
"tools": [
|
||||||
|
"builder_ask_again",
|
||||||
|
"builder_seat_queue",
|
||||||
|
"builder_stuck"
|
||||||
|
],
|
||||||
|
"emits": [
|
||||||
|
"build.asked-again"
|
||||||
|
],
|
||||||
"consumes": [
|
"consumes": [
|
||||||
"mesh-build-go.toolchain.reported",
|
"mesh-build-go.toolchain.reported",
|
||||||
"mesh-build-node.toolchain.reported",
|
"mesh-build-node.toolchain.reported",
|
||||||
@@ -12,7 +20,10 @@
|
|||||||
],
|
],
|
||||||
"state": [
|
"state": [
|
||||||
{ "name": "builds", "history": 5 },
|
{ "name": "builds", "history": 5 },
|
||||||
{ "name": "instances", "history": 1 }
|
{ "name": "hearing", "history": 1 },
|
||||||
|
{ "name": "instances", "history": 1 },
|
||||||
|
{ "name": "queues", "history": 1 },
|
||||||
|
{ "name": "reasks", "history": 1 }
|
||||||
],
|
],
|
||||||
"reads": [
|
"reads": [
|
||||||
"mesh-build-go.steps",
|
"mesh-build-go.steps",
|
||||||
@@ -78,6 +89,17 @@
|
|||||||
"system": "arch",
|
"system": "arch",
|
||||||
"from": "cmd/mesh-delivery-builder",
|
"from": "cmd/mesh-delivery-builder",
|
||||||
"binary": "mesh-delivery-builder"
|
"binary": "mesh-delivery-builder"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tools",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/mesh-delivery-builder-tools",
|
||||||
|
"binary": "mesh-delivery-builder-tools",
|
||||||
|
"loads": [
|
||||||
|
"mesh-delivery-builder-tools"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,16 +7,45 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
"mesh-delivery-checker/internal/check"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
|
||||||
|
var reads atomic.Int64
|
||||||
|
|
||||||
|
// cutAnswer says in the answer that it was cut, and how to narrow it.
|
||||||
|
func cutAnswer(answer map[string]any, cut string) map[string]any {
|
||||||
|
if cut != "" {
|
||||||
|
answer["cut"], answer["note"] = true, cut
|
||||||
|
}
|
||||||
|
return answer
|
||||||
|
}
|
||||||
|
|
||||||
|
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
|
||||||
|
func logged(tools []stdio.Tool) []stdio.Tool {
|
||||||
|
for i := range tools {
|
||||||
|
name, run := tools[i].Name, tools[i].Run
|
||||||
|
tools[i].Run = func(args map[string]any) (any, error) {
|
||||||
|
reads.Store(0)
|
||||||
|
start := time.Now()
|
||||||
|
out, err := run(args)
|
||||||
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tools
|
||||||
|
}
|
||||||
|
|
||||||
func read(s stdio.KeptState) check.StateRead {
|
func read(s stdio.KeptState) check.StateRead {
|
||||||
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||||
|
reads.Add(1)
|
||||||
e, err := s.Get(key)
|
e, err := s.Get(key)
|
||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -25,10 +54,36 @@ func read(s stdio.KeptState) check.StateRead {
|
|||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
|
||||||
|
// (the commit the module's record was computed at); "" when the record is not readable.
|
||||||
|
type runsOf struct{ modules check.StateRead }
|
||||||
|
|
||||||
|
func (r runsOf) CommitOf(module string) string {
|
||||||
|
if r.modules.Get == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
raw, err := r.modules.Get(module)
|
||||||
|
if err != nil || len(raw) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
m, err := delivery.DecodeModule(raw)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return m.Commit
|
||||||
|
}
|
||||||
|
|
||||||
// stateStore is a state the tools write: the tries.
|
// stateStore is a state the tools write: the tries.
|
||||||
type stateStore struct {
|
type stateStore struct {
|
||||||
check.StateRead
|
check.StateRead
|
||||||
create func(key string, value []byte) error
|
create func(key string, value []byte) error
|
||||||
|
entry func(key string) (json.RawMessage, uint64, error)
|
||||||
|
update func(key string, value []byte, revision uint64) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
|
||||||
|
func (s stateStore) Update(key string, value []byte, revision uint64) error {
|
||||||
|
return s.update(key, value, revision)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
||||||
@@ -38,11 +93,12 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
|
|||||||
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
||||||
type Trying struct {
|
type Trying struct {
|
||||||
Tries check.TryStore
|
Tries check.TryStore
|
||||||
|
Ender check.TryEnder
|
||||||
Trunks func() (map[string]string, error)
|
Trunks func() (map[string]string, error)
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
|
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
|
||||||
func TryTools(tr Trying) []stdio.Tool {
|
func TryTools(tr Trying) []stdio.Tool {
|
||||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||||
return []stdio.Tool{
|
return []stdio.Tool{
|
||||||
@@ -50,7 +106,8 @@ func TryTools(tr Trying) []stdio.Tool {
|
|||||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||||
|
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||||
@@ -87,6 +144,28 @@ func TryTools(tr Trying) []stdio.Tool {
|
|||||||
}
|
}
|
||||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||||
}},
|
}},
|
||||||
|
{Name: check.TryEndTool,
|
||||||
|
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||||
|
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||||
|
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||||
|
"takes it within the step bound. A try that ended already is refused.",
|
||||||
|
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||||
|
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||||
|
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
if tr.Ender == nil {
|
||||||
|
return nil, fmt.Errorf("the tries state cannot be written here")
|
||||||
|
}
|
||||||
|
now := time.Now
|
||||||
|
if tr.Now != nil {
|
||||||
|
now = tr.Now
|
||||||
|
}
|
||||||
|
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
|
||||||
|
}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,8 +179,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
|||||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
group, _ := args["group"].(string)
|
group, _ := args["group"].(string)
|
||||||
vs, err := r.Verdicts(group)
|
vs, cut, err := r.Verdicts(group)
|
||||||
return map[string]any{"verdicts": vs}, err
|
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||||
}},
|
}},
|
||||||
{Name: check.PullReadTool,
|
{Name: check.PullReadTool,
|
||||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||||
@@ -122,8 +201,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
|||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
level, _ := args["level"].(string)
|
level, _ := args["level"].(string)
|
||||||
digest, _ := args["digest"].(string)
|
digest, _ := args["digest"].(string)
|
||||||
kept, err := r.Kept(level, digest)
|
kept, cut, err := r.Kept(level, digest)
|
||||||
return map[string]any{"kept": kept}, err
|
return cutAnswer(map[string]any{"kept": kept}, cut), err
|
||||||
}},
|
}},
|
||||||
{Name: check.PreviewsReadTool,
|
{Name: check.PreviewsReadTool,
|
||||||
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||||
@@ -138,15 +217,18 @@ func Tools(r check.Reader) []stdio.Tool {
|
|||||||
if head != "" && len(head) < 7 {
|
if head != "" && len(head) < 7 {
|
||||||
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||||
}
|
}
|
||||||
ps, bad, err := r.Previews(repository, head)
|
ps, bad, cut, err := r.Previews(repository, head)
|
||||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
|
||||||
}},
|
}},
|
||||||
{Name: check.StatusReadTool,
|
{Name: check.StatusReadTool,
|
||||||
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||||
|
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||||
|
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||||
|
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
|
||||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||||
}
|
}
|
||||||
@@ -165,7 +247,9 @@ func main() {
|
|||||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||||
PreviewState: read(stdio.State("previews")),
|
PreviewState: read(stdio.State("previews")),
|
||||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||||
|
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
|
||||||
|
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||||
tries := stdio.State(check.TriesState)
|
tries := stdio.State(check.TriesState)
|
||||||
source := stdio.State("mesh-delivery-gate.source")
|
source := stdio.State("mesh-delivery-gate.source")
|
||||||
trunks := func() (map[string]string, error) {
|
trunks := func() (map[string]string, error) {
|
||||||
@@ -177,14 +261,22 @@ func main() {
|
|||||||
return s.Heads, err
|
return s.Heads, err
|
||||||
}
|
}
|
||||||
r.Trunks = trunks
|
r.Trunks = trunks
|
||||||
trying := Trying{
|
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
_, err := tries.Create(key, json.RawMessage(value))
|
||||||
_, err := tries.Create(key, json.RawMessage(value))
|
return err
|
||||||
return err
|
}, entry: func(key string) (json.RawMessage, uint64, error) {
|
||||||
}},
|
reads.Add(1)
|
||||||
Trunks: trunks,
|
e, err := tries.Get(key)
|
||||||
}
|
if err != nil || e == nil {
|
||||||
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
return e.Value, e.Revision, nil
|
||||||
|
}, update: func(key string, value []byte, revision uint64) error {
|
||||||
|
_, err := tries.Update(key, json.RawMessage(value), revision)
|
||||||
|
return err
|
||||||
|
}}
|
||||||
|
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||||
|
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,13 +53,19 @@ func (m *memStore) Create(key string, value []byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
|
||||||
|
func (m *memStore) Update(key string, value []byte, revision uint64) error {
|
||||||
|
m.vals[key] = value
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
||||||
store := &memStore{vals: map[string]json.RawMessage{}}
|
store := &memStore{vals: map[string]json.RawMessage{}}
|
||||||
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
|
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||||
Trunks: func() (map[string]string, error) {
|
Trunks: func() (map[string]string, error) {
|
||||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||||
}})
|
}})
|
||||||
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
|
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
|
||||||
t.Fatalf("tools = %+v", tools)
|
t.Fatalf("tools = %+v", tools)
|
||||||
}
|
}
|
||||||
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||||
@@ -74,6 +80,14 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
|||||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||||
t.Errorf("read = %s, %v", raw, err)
|
t.Errorf("read = %s, %v", raw, err)
|
||||||
}
|
}
|
||||||
|
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||||
|
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||||
|
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||||
|
t.Errorf("end = %v, %v", ended, err)
|
||||||
|
}
|
||||||
|
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
|
||||||
|
t.Error("an ended try was ended again")
|
||||||
|
}
|
||||||
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
||||||
err.Error() != "no level of novox/hq is checked by the checker" {
|
err.Error() != "no level of novox/hq is checked by the checker" {
|
||||||
t.Errorf("hq: %v", err)
|
t.Errorf("hq: %v", err)
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
"mesh-delivery-checker/internal/check"
|
||||||
)
|
)
|
||||||
@@ -79,6 +80,8 @@ type fakeBuildModules struct {
|
|||||||
js jetstream.JetStream
|
js jetstream.JetStream
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
conflict atomic.Bool
|
conflict atomic.Bool
|
||||||
|
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
|
||||||
|
hold atomic.Value
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
calls map[string]int // by `<module>.<tool>`
|
calls map[string]int // by `<module>.<tool>`
|
||||||
testTool *testing.T
|
testTool *testing.T
|
||||||
@@ -98,6 +101,9 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
|||||||
var v any
|
var v any
|
||||||
switch {
|
switch {
|
||||||
case strings.HasSuffix(tool, "_fetch"):
|
case strings.HasSuffix(tool, "_fetch"):
|
||||||
|
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
|
||||||
|
<-f.ctx.Done()
|
||||||
|
}
|
||||||
name := "go-fetch-clean"
|
name := "go-fetch-clean"
|
||||||
if f.conflict.Load() {
|
if f.conflict.Load() {
|
||||||
name = "go-fetch-conflict"
|
name = "go-fetch-conflict"
|
||||||
@@ -298,6 +304,10 @@ func startBed(t *testing.T) *bed {
|
|||||||
}
|
}
|
||||||
b.tools[module] = kv
|
b.tools[module] = kv
|
||||||
}
|
}
|
||||||
|
// What the controller keeps of the messages a consumer gave up on (issue 734).
|
||||||
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
// The checker's stream of events, which the indexer and the forge bridge hear.
|
// The checker's stream of events, which the indexer and the forge bridge hear.
|
||||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -305,7 +315,8 @@ func startBed(t *testing.T) *bed {
|
|||||||
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
|
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
|
||||||
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
|
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
|
||||||
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
|
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
|
||||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
|
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
|
||||||
|
delivery.TurnsBucket: delivery.TurnsHistory} {
|
||||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -507,9 +518,12 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
|
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
|
||||||
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
|
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
|
||||||
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
|
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
|
||||||
|
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
|
||||||
|
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
|
||||||
|
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||||
b.start(t, nil)
|
b.start(t, nil)
|
||||||
@@ -526,6 +540,42 @@ func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1
|
|||||||
if got := keptCount(t, b); got != kept {
|
if got := keptCount(t, b); got != kept {
|
||||||
t.Errorf("%d kept keys, was %d", got, kept)
|
t.Errorf("%d kept keys, was %d", got, kept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
|
||||||
|
at := time.Now().UTC()
|
||||||
|
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tk, _ := delivery.TurnsKey(repo)
|
||||||
|
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond) // the checker hears the turn
|
||||||
|
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
|
||||||
|
}
|
||||||
|
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.verdict(t, third, ended)
|
||||||
|
|
||||||
|
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
|
||||||
|
tests = b.fake.count("mesh-build-go.build_go_test")
|
||||||
|
const moved = "abababababababababababababababababababab"
|
||||||
|
b.setTrunk(t, map[string]string{repo: moved})
|
||||||
|
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
|
||||||
|
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
|
||||||
|
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
|
||||||
|
}
|
||||||
|
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
|
||||||
|
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
|
||||||
|
}
|
||||||
|
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
|
||||||
|
t.Errorf("the carried status says %q", o.Text)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func keptCount(t *testing.T, b *bed) int {
|
func keptCount(t *testing.T, b *bed) int {
|
||||||
@@ -676,6 +726,49 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
|||||||
t.Errorf("a try wrote %s: %v", name, keys)
|
t.Errorf("a try wrote %s: %v", name, keys)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
|
||||||
|
// message id) ends as error saying so, and its head asked again is a new try.
|
||||||
|
b.fake.hold.Store(head2)
|
||||||
|
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
askID := ""
|
||||||
|
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
keys, _ := steps.Keys(b.ctx)
|
||||||
|
for _, k := range keys {
|
||||||
|
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
|
||||||
|
askID = strings.TrimPrefix(k, "n1.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if askID == "" {
|
||||||
|
t.Fatal("the held try's ask was never taken")
|
||||||
|
}
|
||||||
|
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
|
||||||
|
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
|
||||||
|
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
|
||||||
|
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
|
||||||
|
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
|
||||||
|
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
raw, _ := store.Get(lost)
|
||||||
|
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
|
||||||
|
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
|
||||||
|
}
|
||||||
|
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
|
||||||
|
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
|
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
|
||||||
@@ -702,7 +795,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
|
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
|
||||||
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
|
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
|||||||
if !equal(manifest, announced) {
|
if !equal(manifest, announced) {
|
||||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||||
}
|
}
|
||||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
|
||||||
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
|
||||||
t.Errorf("readable %v, want %v", readable, want)
|
t.Errorf("readable %v, want %v", readable, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-checker/internal/check"
|
||||||
|
)
|
||||||
|
|
||||||
|
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
|
||||||
|
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
|
||||||
|
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
|
||||||
|
// and once again when it opens.
|
||||||
|
type lazyTurns struct {
|
||||||
|
turns *check.Turns
|
||||||
|
open func() error
|
||||||
|
logf func(format string, args ...any)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
opened bool
|
||||||
|
said bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure opens the bucket once it is there; a failure is said once.
|
||||||
|
func (l *lazyTurns) ensure() {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
if l.opened || l.open == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := l.open(); err != nil {
|
||||||
|
if !l.said {
|
||||||
|
l.said = true
|
||||||
|
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
|
||||||
|
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.opened = true
|
||||||
|
if l.said {
|
||||||
|
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holding opens the bucket when it is not yet, then answers what the turns hold.
|
||||||
|
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||||
|
l.ensure()
|
||||||
|
return l.turns.Holding(repositories, heads)
|
||||||
|
}
|
||||||
@@ -112,14 +112,17 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
|||||||
|
|
||||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
|
||||||
var keptStates = []announce.State{
|
var keptStates = []announce.State{
|
||||||
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||||
|
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
|
||||||
|
{Name: "hearing"},
|
||||||
{Name: "kept", Readable: true},
|
{Name: "kept", Readable: true},
|
||||||
{Name: "owed", Readable: true},
|
{Name: "owed", Readable: true},
|
||||||
{Name: "previews", Readable: true},
|
{Name: "previews", Readable: true},
|
||||||
{Name: "tries"},
|
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
|
||||||
|
{Name: "tries", Readable: true},
|
||||||
{Name: "verdicts", Readable: true},
|
{Name: "verdicts", Readable: true},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,17 +155,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
|
||||||
kv, err := js.KeyValue(ctx, name)
|
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
|
||||||
}
|
|
||||||
return kv, nil
|
|
||||||
}
|
|
||||||
buckets := map[string]jetstream.KeyValue{}
|
buckets := map[string]jetstream.KeyValue{}
|
||||||
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
|
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
|
||||||
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
|
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
|
||||||
if buckets[name], err = bucket(name); err != nil {
|
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -198,13 +196,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
tools := &buildrun.ToolchainSet{}
|
tools := &buildrun.ToolchainSet{}
|
||||||
presence := &buildrun.PresenceSet{}
|
presence := &buildrun.PresenceSet{}
|
||||||
trunks := &check.Trunks{}
|
trunks := &check.Trunks{}
|
||||||
|
turns := &check.Turns{}
|
||||||
|
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
|
||||||
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
||||||
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
||||||
asks := buildrun.JetStreamAsks{JS: js}
|
asks := buildrun.JetStreamAsks{JS: js}
|
||||||
|
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
|
||||||
|
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||||
|
// state hearing, which checker_status says (dead_letters).
|
||||||
|
dead := &buildrun.DeadLetters{}
|
||||||
|
hearingKV := process.OptionalBucket(ctx, js, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||||
|
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
|
||||||
|
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||||
|
defer hearing.Stop()
|
||||||
|
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||||
|
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
|
||||||
ch := &check.Checker{
|
ch := &check.Checker{
|
||||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||||
Trunks: trunks,
|
Trunks: trunks,
|
||||||
|
Turns: lazy,
|
||||||
Bound: checkBound,
|
Bound: checkBound,
|
||||||
Link: link,
|
Link: link,
|
||||||
Logf: logger.Printf,
|
Logf: logger.Printf,
|
||||||
@@ -212,12 +223,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||||
// says nothing of the builds, until the checker starts again.
|
// says nothing of the builds, until the checker starts again.
|
||||||
var builds check.BuildStates
|
var builds check.BuildStates
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
|
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
|
|
||||||
case err != nil:
|
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
|
|
||||||
default:
|
|
||||||
builds = buildReader{kvStore{kv}}
|
builds = buildReader{kvStore{kv}}
|
||||||
}
|
}
|
||||||
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
|
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
|
||||||
@@ -238,7 +244,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return cw, nil
|
return cw, nil
|
||||||
}}
|
}}
|
||||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
|
||||||
announce := func(ctx context.Context, event, key string) {
|
announce := func(ctx context.Context, event, key string) {
|
||||||
body, _ := json.Marshal(map[string]string{"key": key})
|
body, _ := json.Marshal(map[string]string{"key": key})
|
||||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||||
@@ -246,8 +252,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
||||||
|
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||||
ch.Judge = newJudge(judgeDeps{
|
ch.Judge = newJudge(judgeDeps{
|
||||||
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
|
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||||
announce: announce, compositions: compositions,
|
announce: announce, compositions: compositions,
|
||||||
@@ -290,55 +297,75 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
|
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
|
||||||
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
|
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
|
||||||
// issue 1912), so the node-engine's have is not watched here.
|
// issue 1912), so the node-engine's have is not watched here.
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil {
|
||||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
|
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again")
|
||||||
case err != nil:
|
} else {
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
|
go func() {
|
||||||
default:
|
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||||
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
|
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||||
if err != nil {
|
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||||
return err
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
go func() {
|
|
||||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
|
||||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
|
||||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
|
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
|
||||||
// yet no trunk is known, so nothing is checked.
|
// yet no trunk is known, so nothing is checked.
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
|
|
||||||
case err != nil:
|
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
|
||||||
default:
|
|
||||||
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
|
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
|
||||||
if e.Key() == delivery.SourceMainKey {
|
if e.Key() == delivery.SourceMainKey {
|
||||||
trunks.Put(e.Value(), !opOf(e))
|
trunks.Put(e.Value(), !opOf(e))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
|
||||||
|
} else {
|
||||||
|
go func() {
|
||||||
|
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||||
|
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||||
|
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
|
||||||
|
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
|
||||||
|
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
|
||||||
|
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
|
||||||
|
lazy.open = func() error {
|
||||||
|
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
|
||||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}()
|
}()
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
lazy.ensure()
|
||||||
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
||||||
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
|
||||||
|
before := repos.CommitOf(check.NodeEngineModule)
|
||||||
|
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
|
||||||
|
if after := repos.CommitOf(check.NodeEngineModule); after != before {
|
||||||
|
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
|
||||||
|
}
|
||||||
|
})
|
||||||
}()
|
}()
|
||||||
each := func(e jetstream.KeyValueEntry) {
|
each := func(e jetstream.KeyValueEntry) {
|
||||||
if opOf(e) {
|
if opOf(e) {
|
||||||
|
|||||||
@@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||||
w, err := settle(ctx, kv, each)
|
w, err := settle(ctx, kv, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||||
|
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
|
return errs, nil
|
||||||
}
|
}
|
||||||
if gate.Settled() {
|
if gate.Settled() {
|
||||||
changed(ctx)
|
changed(ctx)
|
||||||
|
|||||||
@@ -3,11 +3,19 @@ package main
|
|||||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
@@ -89,6 +97,51 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
if n := len(set.Entries("typescript")); n != 2 {
|
if n := len(set.Entries("typescript")); n != 2 {
|
||||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||||
|
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||||
|
// is an error naming the bucket and its grant.
|
||||||
|
var logs bytes.Buffer
|
||||||
|
logger := log.New(&logs, "", 0)
|
||||||
|
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||||
|
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||||
|
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||||
|
}
|
||||||
|
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||||
|
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||||
|
t.Errorf("the refusals were not said: %q", logs.String())
|
||||||
|
}
|
||||||
|
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||||
|
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||||
|
type refusingBus struct {
|
||||||
|
refuseOpen bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||||
|
if r.refuseOpen {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
return refusedWatch{err: r.err}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type refusedWatch struct {
|
||||||
|
jetstream.KeyValue
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||||
|
return nil, r.err
|
||||||
}
|
}
|
||||||
|
|
||||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -63,11 +65,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
|||||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||||
return nil
|
return nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||||
|
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
w.watched[module] = true
|
w.watched[module] = true
|
||||||
go func() {
|
go func() {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026134035-21252194d530
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026134035-21252194d530 h1:NwCb+gOh0UYqlyG6hG2/jffE/Gd0c7YWd4iJlaCB0lU=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026134035-21252194d530/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -0,0 +1,355 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
|
||||||
|
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
|
||||||
|
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
|
||||||
|
|
||||||
|
// turnOf is a turn's value on a repository for a delivery of the heads hash.
|
||||||
|
func turnOf(t testing.TB, repo, group, heads string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||||
|
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func turnKey(t testing.TB, repo string) string {
|
||||||
|
t.Helper()
|
||||||
|
k, err := delivery.TurnsKey(repo)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
|
||||||
|
func waitFor(t testing.TB, what string, ok func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("%s did not happen", what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
|
||||||
|
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
|
||||||
|
p := pullOf(repo, number, head)
|
||||||
|
p.Modules = []string{"m"}
|
||||||
|
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOneCheckPerDelivery(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
for name, run := range map[string]func(t *testing.T){
|
||||||
|
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
|
||||||
|
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
|
||||||
|
j := passJudge()
|
||||||
|
r := newRig(t, j)
|
||||||
|
turns := &Turns{}
|
||||||
|
r.ch.Turns = turns
|
||||||
|
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 0 || len(r.verdicts.vals) != 0 {
|
||||||
|
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
|
||||||
|
}
|
||||||
|
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
|
||||||
|
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
|
||||||
|
}
|
||||||
|
if !turns.Put(turnKey(t, repoA), nil, true) {
|
||||||
|
t.Fatal("the turn's delete changed nothing")
|
||||||
|
}
|
||||||
|
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||||
|
t.Errorf("judged %d times after the turn went, want once", j.count())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
|
||||||
|
j := passJudge()
|
||||||
|
r := newRig(t, j)
|
||||||
|
turns := &Turns{}
|
||||||
|
r.ch.Turns = turns
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
c, _ := delivery.DecodeCandidate(cand)
|
||||||
|
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
|
||||||
|
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 1 {
|
||||||
|
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
|
||||||
|
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
|
||||||
|
release, started := make(chan struct{}), make(chan struct{})
|
||||||
|
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
|
||||||
|
close(started)
|
||||||
|
<-release
|
||||||
|
return Outcome{Verdict: passing()}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
turns := &Turns{}
|
||||||
|
r.ch.Turns = turns
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
<-started
|
||||||
|
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||||
|
r.ch.Reconsider(ctx)
|
||||||
|
close(release)
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||||
|
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
|
||||||
|
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
|
||||||
|
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
|
||||||
|
var r *rig
|
||||||
|
var kinds []string
|
||||||
|
var from []map[string]string
|
||||||
|
judged := make(chan struct{})
|
||||||
|
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||||
|
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||||
|
if call == 1 {
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||||
|
r.ch.TrunkMoved(ctx) // as the watch of the source says it
|
||||||
|
close(judged)
|
||||||
|
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||||
|
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
|
||||||
|
}
|
||||||
|
return Outcome{Verdict: passing()}, nil
|
||||||
|
}}
|
||||||
|
r = newRig(t, j)
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
<-judged
|
||||||
|
r.ch.Wait()
|
||||||
|
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||||
|
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
|
||||||
|
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
|
||||||
|
}
|
||||||
|
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||||
|
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
|
||||||
|
}
|
||||||
|
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||||
|
t.Errorf("the status after the carry is %q, want success", s)
|
||||||
|
}
|
||||||
|
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
|
||||||
|
t.Error("the verdict key was made again: a carry rewrites it")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
|
||||||
|
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
|
||||||
|
var kept []bool
|
||||||
|
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||||
|
kept = append(kept, c.Composition != nil)
|
||||||
|
v := passing()
|
||||||
|
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
|
||||||
|
return Outcome{Verdict: v}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||||
|
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
|
||||||
|
r.ch.Wait()
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
|
||||||
|
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
|
||||||
|
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
|
||||||
|
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
|
||||||
|
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
|
||||||
|
var r *rig
|
||||||
|
var from map[string]string
|
||||||
|
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||||
|
if call == 2 {
|
||||||
|
from = c.CarriedFrom
|
||||||
|
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||||
|
t.Errorf("the status during the sibling's carry is %q, want pending", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
|
||||||
|
}}
|
||||||
|
r = newRig(t, j)
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
|
||||||
|
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
|
||||||
|
}
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||||
|
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
|
||||||
|
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
|
||||||
|
}
|
||||||
|
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||||
|
t.Errorf("the status after the carry is %q, want success", s)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
|
||||||
|
// cleanly asks a whole check, not a carry.
|
||||||
|
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
|
||||||
|
var kinds []string
|
||||||
|
var from []map[string]string
|
||||||
|
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||||
|
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||||
|
if call == 2 {
|
||||||
|
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
|
||||||
|
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
|
||||||
|
}
|
||||||
|
return Outcome{Verdict: passing()}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
|
||||||
|
r.ch.Wait()
|
||||||
|
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
|
||||||
|
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
|
||||||
|
}
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
|
||||||
|
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
|
||||||
|
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
|
||||||
|
"the line of a carry says where it was carried from": func(t *testing.T) {
|
||||||
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
|
v := passVerdict()
|
||||||
|
v.Trunk = map[string]string{repoA: trunkA}
|
||||||
|
v.Beside = map[string]string{catalogueRepository: trunkS2}
|
||||||
|
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
|
||||||
|
line := Line(v, cs, p, nil, nil, nil)
|
||||||
|
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
|
||||||
|
t.Errorf("the carried line = %q", line)
|
||||||
|
}
|
||||||
|
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
|
||||||
|
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
|
||||||
|
}
|
||||||
|
cs.CarriedFrom = map[string]string{repoA: trunkA2}
|
||||||
|
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
|
||||||
|
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
|
||||||
|
t.Errorf("the pending line of a carry = %q", pending)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
|
||||||
|
"two merges waiting in one turn each get their carry": func(t *testing.T) {
|
||||||
|
var r *rig
|
||||||
|
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
|
||||||
|
r = newRig(t, j)
|
||||||
|
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fa", candA)
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fc", candC)
|
||||||
|
r.ch.Wait()
|
||||||
|
a, _ := delivery.DecodeCandidate(candA)
|
||||||
|
c, _ := delivery.DecodeCandidate(candC)
|
||||||
|
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||||
|
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
|
||||||
|
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
turns := &Turns{}
|
||||||
|
turns.Put(turnKey(t, repoA), turn, false)
|
||||||
|
r.ch.Turns = turns
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
for _, cand := range [][]byte{candA, candC} {
|
||||||
|
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
|
||||||
|
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||||
|
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
|
||||||
|
"a new running node-engine is carried": func(t *testing.T) {
|
||||||
|
runs := runsAt{nodeEngineModule: trunkS}
|
||||||
|
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||||
|
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
r.ch.Runs = runs
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
|
||||||
|
r.ch.Wait()
|
||||||
|
runs[nodeEngineModule] = trunkS2
|
||||||
|
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
|
||||||
|
r.ch.Wait()
|
||||||
|
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||||
|
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
|
||||||
|
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §5: a carry that fails names the move it failed on.
|
||||||
|
"a carry that fails names the move": func(t *testing.T) {
|
||||||
|
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||||
|
if call == 1 {
|
||||||
|
return Outcome{Verdict: passing()}, nil
|
||||||
|
}
|
||||||
|
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||||
|
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
|
||||||
|
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
|
||||||
|
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(name, run)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
|
||||||
|
func checkWordsOf(line string) (string, string) {
|
||||||
|
for _, part := range strings.Split(line, " | ") {
|
||||||
|
f := strings.Fields(part)
|
||||||
|
for i := 0; i+2 < len(f); i++ {
|
||||||
|
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
|
||||||
|
return f[i], f[i+2]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
@@ -127,6 +127,9 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
withComposition(&v, comp)
|
withComposition(&v, comp)
|
||||||
|
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||||
|
v.Composition.FPSet = cs.FPSet
|
||||||
|
}
|
||||||
data, err := delivery.EncodeVerdict(v)
|
data, err := delivery.EncodeVerdict(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
||||||
|
|||||||
@@ -12,8 +12,18 @@ type BesideRef struct {
|
|||||||
Repository string `json:"repository"`
|
Repository string `json:"repository"`
|
||||||
Commit string `json:"commit"`
|
Commit string `json:"commit"`
|
||||||
Ref string `json:"ref,omitempty"`
|
Ref string `json:"ref,omitempty"`
|
||||||
|
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
|
||||||
|
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
|
||||||
|
// group's member at its head do not move with a trunk.
|
||||||
|
atTrunk bool
|
||||||
|
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
|
||||||
|
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
|
||||||
|
runs bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
|
||||||
|
const NodeEngineModule = nodeEngineModule
|
||||||
|
|
||||||
const (
|
const (
|
||||||
catalogueRepository = "novox/mesh-catalog"
|
catalogueRepository = "novox/mesh-catalog"
|
||||||
nodeEngineRepo = "novox/mesh-host"
|
nodeEngineRepo = "novox/mesh-host"
|
||||||
@@ -51,18 +61,21 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
|||||||
switch {
|
switch {
|
||||||
case repo == catalogueRepository:
|
case repo == catalogueRepository:
|
||||||
if head := heads[repo]; head != "" {
|
if head := heads[repo]; head != "" {
|
||||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
|
||||||
}
|
}
|
||||||
case repo == nodeEngineRepo:
|
case repo == nodeEngineRepo:
|
||||||
commit := ""
|
commit := ""
|
||||||
if j.Runs != nil {
|
if j.Runs != nil {
|
||||||
commit = j.Runs.CommitOf(nodeEngineModule)
|
commit = j.Runs.CommitOf(nodeEngineModule)
|
||||||
}
|
}
|
||||||
|
atTrunk := false
|
||||||
if commit == "" {
|
if commit == "" {
|
||||||
commit = heads[repo]
|
commit, atTrunk = heads[repo], true
|
||||||
}
|
}
|
||||||
if commit != "" {
|
if commit != "" {
|
||||||
out = append(out, BesideRef{Repository: repo, Commit: commit})
|
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
|
||||||
|
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
|
||||||
|
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
|
||||||
}
|
}
|
||||||
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
|
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
|
||||||
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
|
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
|
||||||
@@ -82,7 +95,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
|||||||
if member := c.member(repo); member != nil {
|
if member := c.member(repo); member != nil {
|
||||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||||
} else if head := heads[repo]; head != "" {
|
} else if head := heads[repo]; head != "" {
|
||||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
|
||||||
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
|
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
|
||||||
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
|
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
|
||||||
// checker that cannot place it says so, where placing none left that test to skip.
|
// checker that cannot place it says so, where placing none left that test to skip.
|
||||||
@@ -108,3 +121,13 @@ func refOf(number int) string {
|
|||||||
}
|
}
|
||||||
return "refs/pull/" + strconv.Itoa(number) + "/head"
|
return "refs/pull/" + strconv.Itoa(number) + "/head"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
|
||||||
|
// verdict names as Beside (ADR 0548 §6).
|
||||||
|
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
|
||||||
|
for _, r := range refs {
|
||||||
|
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
|
||||||
|
into[r.Repository] = r.Commit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
||||||
|
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
|
||||||
|
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
|
||||||
|
// through mesh-sdk (rules-are-weighed).
|
||||||
|
const ReadWorkers = 16
|
||||||
|
|
||||||
|
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
|
||||||
|
const MaxReads = 1000
|
||||||
|
|
||||||
|
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
|
||||||
|
// further read starts.
|
||||||
|
func parallel(n int, read func(i int) error) error {
|
||||||
|
var (
|
||||||
|
next atomic.Int64
|
||||||
|
failed atomic.Bool
|
||||||
|
once sync.Once
|
||||||
|
first error
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for !failed.Load() {
|
||||||
|
i := int(next.Add(1) - 1)
|
||||||
|
if i >= n {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := read(i); err != nil {
|
||||||
|
once.Do(func() { first = err })
|
||||||
|
failed.Store(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
|
||||||
|
func capKeys(keys []string, narrow string) ([]string, string) {
|
||||||
|
if len(keys) <= MaxReads {
|
||||||
|
return keys, ""
|
||||||
|
}
|
||||||
|
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
|
||||||
|
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
|
||||||
|
func newestFirst(keys []string) {
|
||||||
|
id := func(k string) (string, bool) {
|
||||||
|
_, id, err := delivery.ParseStepKey(k)
|
||||||
|
return id, err == nil
|
||||||
|
}
|
||||||
|
sort.Slice(keys, func(i, j int) bool {
|
||||||
|
a, aok := id(keys[i])
|
||||||
|
b, bok := id(keys[j])
|
||||||
|
switch {
|
||||||
|
case aok != bok:
|
||||||
|
return aok
|
||||||
|
case len(a) != len(b):
|
||||||
|
return len(a) > len(b)
|
||||||
|
case a != b:
|
||||||
|
return a > b
|
||||||
|
}
|
||||||
|
return keys[i] > keys[j]
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -44,11 +44,35 @@ type Case struct {
|
|||||||
|
|
||||||
notesMu sync.Mutex
|
notesMu sync.Mutex
|
||||||
notes []string
|
notes []string
|
||||||
|
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
|
||||||
|
asks []string
|
||||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||||
Progress *Progress
|
Progress *Progress
|
||||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
|
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||||
// seats' recheck stream, every other check on check (ADR 0536 §4).
|
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
|
||||||
Kind string
|
Kind string
|
||||||
|
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
|
||||||
|
// nil for a whole check.
|
||||||
|
CarriedFrom map[string]string
|
||||||
|
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
|
||||||
|
FPSet string
|
||||||
|
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
|
||||||
|
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
|
||||||
|
Composition *delivery.Composition
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asked records the id of an ask the check put.
|
||||||
|
func (c *Case) Asked(id string) {
|
||||||
|
c.notesMu.Lock()
|
||||||
|
c.asks = append(c.asks, id)
|
||||||
|
c.notesMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asks are the ids of the asks the check put, in order.
|
||||||
|
func (c *Case) Asks() []string {
|
||||||
|
c.notesMu.Lock()
|
||||||
|
defer c.notesMu.Unlock()
|
||||||
|
return append([]string(nil), c.asks...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Progress is the step a check is at, written by the check and read when its bound passes.
|
// Progress is the step a check is at, written by the check and read when its bound passes.
|
||||||
@@ -80,7 +104,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
|
||||||
for _, p := range c.Pulls {
|
for _, p := range c.Pulls {
|
||||||
switch head, known := trunk[p.Repository]; {
|
switch head, known := trunk[p.Repository]; {
|
||||||
case !known || head == "":
|
case !known || head == "":
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -42,6 +43,9 @@ type Outcome struct {
|
|||||||
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
|
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
|
||||||
// that changes none is not named.
|
// that changes none is not named.
|
||||||
Judged map[string]string
|
Judged map[string]string
|
||||||
|
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
|
||||||
|
// a move of one is carried.
|
||||||
|
Beside map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
||||||
@@ -65,6 +69,14 @@ type Checker struct {
|
|||||||
Owed KV
|
Owed KV
|
||||||
Judge Judge
|
Judge Judge
|
||||||
Trunks interface{ Heads() map[string]string }
|
Trunks interface{ Heads() map[string]string }
|
||||||
|
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
|
||||||
|
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
|
||||||
|
Runs interface{ CommitOf(module string) string }
|
||||||
|
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
|
||||||
|
// waiting delivery's own heads; nil holds none.
|
||||||
|
Turns interface {
|
||||||
|
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
|
||||||
|
}
|
||||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||||
Bound time.Duration
|
Bound time.Duration
|
||||||
// Link is the base of the address a status points at; the verdict key follows.
|
// Link is the base of the address a status points at; the verdict key follows.
|
||||||
@@ -83,12 +95,13 @@ type Checker struct {
|
|||||||
tried map[string]bool // by verdict key: looked at since this process started
|
tried map[string]bool // by verdict key: looked at since this process started
|
||||||
epoch uint64 // changes of the nodes seen since this process started
|
epoch uint64 // changes of the nodes seen since this process started
|
||||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||||
|
held map[string]string // by verdict key: the turn that holds its check off, as last said
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ch *Checker) init() {
|
func (ch *Checker) init() {
|
||||||
ch.once.Do(func() {
|
ch.once.Do(func() {
|
||||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||||
ch.nodesFail = map[string]uint64{}
|
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -191,7 +204,7 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
|||||||
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||||
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
||||||
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
||||||
cs.Kind = ch.kindOf(ctx, cs)
|
ch.prepare(ctx, cs)
|
||||||
ch.try(ctx, key, cs)
|
ch.try(ctx, key, cs)
|
||||||
ch.mu.Lock()
|
ch.mu.Lock()
|
||||||
more := ch.again[cs.Key]
|
more := ch.again[cs.Key]
|
||||||
@@ -211,29 +224,130 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
||||||
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
|
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
|
||||||
|
// §5), on the recheck stream, pulled after.
|
||||||
const (
|
const (
|
||||||
KindNew = "new"
|
KindNew = "new"
|
||||||
KindNodes = "nodes"
|
KindNodes = "nodes"
|
||||||
KindTrunk = "trunk"
|
KindTrunk = "trunk"
|
||||||
)
|
)
|
||||||
|
|
||||||
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
|
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
|
||||||
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
|
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
|
||||||
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
|
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
|
||||||
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
|
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
|
||||||
|
// to do.
|
||||||
|
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||||
|
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
|
||||||
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
return KindNew
|
return
|
||||||
}
|
}
|
||||||
v, err := delivery.DecodeVerdict(value)
|
v, err := delivery.DecodeVerdict(value)
|
||||||
switch {
|
switch {
|
||||||
case err != nil || v.State == delivery.VerdictRunning:
|
case err != nil || v.State == delivery.VerdictRunning:
|
||||||
return KindNew
|
return
|
||||||
case IsNodesFailure(v):
|
case IsNodesFailure(v):
|
||||||
return KindNodes
|
cs.Kind = KindNodes
|
||||||
|
return
|
||||||
|
case IsConflictFailure(v):
|
||||||
|
return // a head git refused: its next check is whole (ADR 0548 §7)
|
||||||
}
|
}
|
||||||
return KindTrunk
|
cs.Kind = KindTrunk
|
||||||
|
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
|
||||||
|
cs.CarriedFrom = judgedOn(v)
|
||||||
|
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
|
||||||
|
kept := *c
|
||||||
|
cs.Composition = &kept
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
|
||||||
|
// read at its trunk's head (ADR 0548 §6).
|
||||||
|
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
|
||||||
|
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
|
||||||
|
}
|
||||||
|
|
||||||
|
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
|
||||||
|
// mesh runs when the indexer names one, as besideFor places them.
|
||||||
|
func (ch *Checker) besideNow() map[string]string {
|
||||||
|
return besideAt(ch.heads(), ch.Runs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
|
||||||
|
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
|
||||||
|
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
|
||||||
|
if runs != nil {
|
||||||
|
if c := runs.CommitOf(nodeEngineModule); c != "" {
|
||||||
|
out := make(map[string]string, len(now)+1)
|
||||||
|
for r, h := range now {
|
||||||
|
out[r] = h
|
||||||
|
}
|
||||||
|
out[nodeEngineRepo] = c
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return now
|
||||||
|
}
|
||||||
|
|
||||||
|
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
|
||||||
|
func besideMoved(beside, now map[string]string) bool {
|
||||||
|
for r, c := range beside {
|
||||||
|
if h := now[r]; h != "" && h != c {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
|
||||||
|
func judgedOn(v delivery.Verdict) map[string]string {
|
||||||
|
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
|
||||||
|
for r, c := range v.Beside {
|
||||||
|
out[r] = c
|
||||||
|
}
|
||||||
|
for r, c := range v.Trunk {
|
||||||
|
out[r] = c
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
|
||||||
|
// repositories the new verdict still judges or reads; nil when none of them moved.
|
||||||
|
func carried(from map[string]string, v delivery.Verdict) map[string]string {
|
||||||
|
out, moved := map[string]string{}, false
|
||||||
|
for r, c := range from {
|
||||||
|
now, ok := v.Trunk[r]
|
||||||
|
if !ok {
|
||||||
|
now, ok = v.Beside[r]
|
||||||
|
}
|
||||||
|
if ok && now != "" && c != "" {
|
||||||
|
out[r] = c
|
||||||
|
moved = moved || now != c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !moved {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
|
||||||
|
// repository, sorted.
|
||||||
|
func moveWords(from map[string]string, v delivery.Verdict) string {
|
||||||
|
var moves []string
|
||||||
|
for r, c := range from {
|
||||||
|
now, ok := v.Trunk[r]
|
||||||
|
if !ok {
|
||||||
|
now = v.Beside[r]
|
||||||
|
}
|
||||||
|
if now != "" && now != c {
|
||||||
|
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(moves)
|
||||||
|
return strings.Join(moves, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
||||||
@@ -265,6 +379,9 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
|||||||
return nil, fmt.Errorf("%s: %w", cs.Key, err)
|
return nil, fmt.Errorf("%s: %w", cs.Key, err)
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
|
if ch.heldByTurn(cs) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
|
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
|
||||||
if errors.Is(err, ErrConflict) {
|
if errors.Is(err, ErrConflict) {
|
||||||
return nil, nil // another checker made it first
|
return nil, nil // another checker made it first
|
||||||
@@ -285,7 +402,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
|||||||
return &claimed{rev}, nil
|
return &claimed{rev}, nil
|
||||||
}
|
}
|
||||||
return nil, nil
|
return nil, nil
|
||||||
case !sameTrunk(cur.Trunk, cs.Trunk):
|
case ch.stale(cur, cs):
|
||||||
retake = true
|
retake = true
|
||||||
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
|
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
|
||||||
ch.mu.Lock()
|
ch.mu.Lock()
|
||||||
@@ -293,7 +410,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
|||||||
ch.mu.Unlock()
|
ch.mu.Unlock()
|
||||||
retake = !ok || epoch > at
|
retake = !ok || epoch > at
|
||||||
}
|
}
|
||||||
if !retake {
|
if !retake || ch.heldByTurn(cs) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
|
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
|
||||||
@@ -306,6 +423,59 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
|||||||
return &claimed{rev}, nil
|
return &claimed{rev}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
|
||||||
|
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
|
||||||
|
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
|
||||||
|
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
|
||||||
|
func (ch *Checker) heldByTurn(cs *Case) bool {
|
||||||
|
if ch.Turns == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
repos := make([]string, 0, len(cs.Trunk))
|
||||||
|
for r := range cs.Trunk {
|
||||||
|
repos = append(repos, r)
|
||||||
|
}
|
||||||
|
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
|
||||||
|
why := ""
|
||||||
|
if held {
|
||||||
|
why = repo + " " + w.Group + " " + w.Heads
|
||||||
|
}
|
||||||
|
ch.mu.Lock()
|
||||||
|
said := ch.held[cs.Key]
|
||||||
|
if why == "" {
|
||||||
|
delete(ch.held, cs.Key)
|
||||||
|
} else {
|
||||||
|
ch.held[cs.Key] = why
|
||||||
|
}
|
||||||
|
ch.mu.Unlock()
|
||||||
|
if held && said != why {
|
||||||
|
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
|
||||||
|
cs.Key, w.Group, w.Heads, repo)
|
||||||
|
}
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
|
||||||
|
func (ch *Checker) Held() map[string]string {
|
||||||
|
ch.init()
|
||||||
|
ch.mu.Lock()
|
||||||
|
defer ch.mu.Unlock()
|
||||||
|
out := make(map[string]string, len(ch.held))
|
||||||
|
for k, v := range ch.held {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
|
||||||
|
const ConflictWords = "does not merge into the trunk: files in conflict: "
|
||||||
|
|
||||||
|
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
|
||||||
|
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
|
||||||
|
func IsConflictFailure(v delivery.Verdict) bool {
|
||||||
|
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
|
||||||
|
}
|
||||||
|
|
||||||
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
||||||
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
||||||
// seats' streams (ADR 0525 §6, issue 1912).
|
// seats' streams (ADR 0525 §6, issue 1912).
|
||||||
@@ -332,11 +502,41 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
|||||||
began := ch.now()
|
began := ch.now()
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
out := ch.judge(ctx, cs)
|
out := ch.judge(ctx, cs)
|
||||||
|
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
|
||||||
|
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
|
||||||
|
for try := 1; IsLostAsk(out.Verdict) && ctx.Err() == nil; try++ {
|
||||||
|
if try >= LostAskTries {
|
||||||
|
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
|
||||||
|
" times and given up each time: " + UnheardWords)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
|
||||||
|
out = ch.judge(ctx, cs)
|
||||||
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
v := out.Verdict
|
v := out.Verdict
|
||||||
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
|
||||||
|
for r, c := range out.Beside {
|
||||||
|
if _, merged := cs.Trunk[r]; !merged && c != "" {
|
||||||
|
if v.Beside == nil {
|
||||||
|
v.Beside = map[string]string{}
|
||||||
|
}
|
||||||
|
v.Beside[r] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cs.CarriedFrom != nil {
|
||||||
|
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
|
||||||
|
switch v.State {
|
||||||
|
case delivery.VerdictPass:
|
||||||
|
v.CarriedFrom = carried(cs.CarriedFrom, v)
|
||||||
|
case delivery.VerdictFail:
|
||||||
|
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
|
||||||
|
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
||||||
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
|
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
|
||||||
}
|
}
|
||||||
@@ -358,11 +558,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
|||||||
ch.nodesFail[cs.Key] = epoch
|
ch.nodesFail[cs.Key] = epoch
|
||||||
ch.mu.Unlock()
|
ch.mu.Unlock()
|
||||||
}
|
}
|
||||||
if ch.movedSince(cs) {
|
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
|
||||||
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
||||||
// check of the new trunk, which Reconsider starts, ends.
|
// check of the new trunk, which Reconsider starts, ends.
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
ch.owePending(ctx, cs, v)
|
||||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
|
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
ch.writeOwed(ctx, cs, v, out)
|
ch.writeOwed(ctx, cs, v, out)
|
||||||
@@ -502,14 +702,37 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
|
|||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
continue // never judged: no status to put out of date
|
continue // never judged: no status to put out of date
|
||||||
}
|
}
|
||||||
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
|
v, err := delivery.DecodeVerdict(value)
|
||||||
|
if err == nil && !ch.stale(v, cs) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if err == nil && v.State == delivery.VerdictPass {
|
||||||
|
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
|
||||||
|
}
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
}
|
}
|
||||||
ch.Reconsider(ctx)
|
ch.Reconsider(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
|
||||||
|
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
|
||||||
|
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
|
||||||
|
next := judged
|
||||||
|
ckey, err := delivery.CandidatesKey(v.Group)
|
||||||
|
ch.mu.Lock()
|
||||||
|
c, ok := ch.known[ckey]
|
||||||
|
ch.mu.Unlock()
|
||||||
|
if err == nil && ok {
|
||||||
|
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
|
||||||
|
next = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v.State == delivery.VerdictPass {
|
||||||
|
next.CarriedFrom = judgedOn(v)
|
||||||
|
}
|
||||||
|
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
|
}
|
||||||
|
|
||||||
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
|
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
|
||||||
func (ch *Checker) movedSince(cs *Case) bool {
|
func (ch *Checker) movedSince(cs *Case) bool {
|
||||||
now := ch.heads()
|
now := ch.heads()
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
|||||||
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
||||||
b.ex.Compositions = f
|
b.ex.Compositions = f
|
||||||
cs := caseOf(t, pullOf(repoA, 7, head1))
|
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
cs.FPSet = "fps-of-the-candidate"
|
||||||
v := b.judge(t, cs).Verdict
|
v := b.judge(t, cs).Verdict
|
||||||
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
|
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
|
||||||
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
|
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
|
||||||
@@ -38,6 +39,15 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
|||||||
if !compositionMentions(v, "passed") {
|
if !compositionMentions(v, "passed") {
|
||||||
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
|
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
|
||||||
}
|
}
|
||||||
|
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
|
||||||
|
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
|
||||||
|
}
|
||||||
|
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
|
||||||
|
carry := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
|
||||||
|
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
|
||||||
|
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func compositionMentions(v delivery.Verdict, word string) bool {
|
func compositionMentions(v delivery.Verdict, word string) bool {
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// failingResult is a level's test result in which the named tests failed, each in its own package.
|
||||||
|
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
|
||||||
|
t.Helper()
|
||||||
|
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||||
|
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
|
||||||
|
return v, st
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
|
||||||
|
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
|
||||||
|
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
|
||||||
|
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
|
||||||
|
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
|
||||||
|
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
|
||||||
|
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
|
||||||
|
for name, tests := range map[string][]json.RawMessage{
|
||||||
|
"one result": {failingResult(t, 10, a, b)},
|
||||||
|
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
|
||||||
|
} {
|
||||||
|
v, st := judgeLevel(t, tests...)
|
||||||
|
if v == nil {
|
||||||
|
t.Fatalf("%s: the level passed", name)
|
||||||
|
}
|
||||||
|
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
|
||||||
|
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||||
|
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||||
|
}
|
||||||
|
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
|
||||||
|
t.Errorf("%s: verdict %+v", name, v)
|
||||||
|
}
|
||||||
|
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
|
||||||
|
t.Errorf("%s: %d tests counted, want all 10", name, run)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
|
||||||
|
t.Errorf("%s: the verdict is not written: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
|
||||||
|
// failed, and each entry's line is cut to MaxFailureLine.
|
||||||
|
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
||||||
|
var fs []Failure
|
||||||
|
for i := range 60 {
|
||||||
|
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
|
||||||
|
}
|
||||||
|
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
|
||||||
|
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
|
||||||
|
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
|
||||||
|
}
|
||||||
|
if len(v.Failures[0].Words) > MaxFailureLine {
|
||||||
|
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
|
||||||
|
}
|
||||||
|
if n := failedCount(*v); n != 60 {
|
||||||
|
t.Errorf("failedCount = %d, want 60", n)
|
||||||
|
}
|
||||||
|
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
|
||||||
|
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
|
||||||
|
t.Errorf("line %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
||||||
|
// before.
|
||||||
|
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
|
||||||
|
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
||||||
|
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
||||||
|
t.Fatalf("%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -116,6 +116,9 @@ type bench struct {
|
|||||||
fetch Fetched
|
fetch Fetched
|
||||||
digest map[string]Digested // by level
|
digest map[string]Digested // by level
|
||||||
test func(c stepCall) (any, error)
|
test func(c stepCall) (any, error)
|
||||||
|
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
|
||||||
|
// one (issue 890); an ask naming none runs on shanks, which then has none.
|
||||||
|
declines int
|
||||||
}
|
}
|
||||||
|
|
||||||
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
|
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
|
||||||
@@ -151,8 +154,17 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
|
|||||||
if module == buildrun.GoBuildModule {
|
if module == buildrun.GoBuildModule {
|
||||||
steps.Digest = tool("digest")
|
steps.Digest = tool("digest")
|
||||||
}
|
}
|
||||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
|
||||||
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||||
|
if len(a.Needs) > 0 {
|
||||||
|
if b.declines > 0 {
|
||||||
|
b.declines--
|
||||||
|
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
|
||||||
|
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
|
||||||
|
Reason: "declined", Result: result}
|
||||||
|
}
|
||||||
|
steps.Has = func(string) bool { return true }
|
||||||
|
}
|
||||||
|
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||||
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
|
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||||
@@ -276,6 +288,49 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
|||||||
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
||||||
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
|
||||||
|
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
|
||||||
|
// the package alone, its kept passes not counted twice.
|
||||||
|
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
|
||||||
|
// placed ask that ran more than its unmet packages would run and count it again.
|
||||||
|
b = newBench(t)
|
||||||
|
b.declines = 1
|
||||||
|
outside := 0
|
||||||
|
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
|
||||||
|
d := echoBeside(b, b.digest[c.args["level"].(string)])
|
||||||
|
if d.Level == "integration" {
|
||||||
|
outside++
|
||||||
|
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
|
||||||
|
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
|
||||||
|
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
|
asks := b.asks.Asks()
|
||||||
|
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
|
||||||
|
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
|
||||||
|
}
|
||||||
|
for _, a := range asks[1:] {
|
||||||
|
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
|
||||||
|
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tests = b.steps.tool("test")
|
||||||
|
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
|
||||||
|
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
|
||||||
|
}
|
||||||
|
slow := captured[Tested](t, "go-test-slow")
|
||||||
|
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
|
||||||
|
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
|
||||||
|
}
|
||||||
|
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
|
||||||
|
b.kept, b.declines = newMemKV(), 1000
|
||||||
|
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
|
||||||
|
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
|
||||||
|
t.Errorf("with no node having the need: %+v", v)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
||||||
@@ -341,7 +396,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
|
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
|
||||||
|
// after it (so one check names every failure), and no later level runs.
|
||||||
|
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
||||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
||||||
@@ -357,8 +414,8 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
|
|||||||
t.Fatalf("verdict = %+v", out.Verdict)
|
t.Fatalf("verdict = %+v", out.Verdict)
|
||||||
}
|
}
|
||||||
tests := b.steps.tool("test")
|
tests := b.steps.tool("test")
|
||||||
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
|
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
|
||||||
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
|
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
|
||||||
}
|
}
|
||||||
if n := len(b.steps.tool("digest")); n != 1 {
|
if n := len(b.steps.tool("digest")); n != 1 {
|
||||||
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
|
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
|
||||||
@@ -618,9 +675,8 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
|
|||||||
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
|
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
|
||||||
func echoBeside(b *bench, d Digested) Digested {
|
func echoBeside(b *bench, d Digested) Digested {
|
||||||
var lines []string
|
var lines []string
|
||||||
for _, f := range b.steps.tool("fetch") {
|
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again)
|
||||||
refs, _ := refsOf(f.args["beside"]), true
|
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) {
|
||||||
for _, r := range refs {
|
|
||||||
lines = append(lines, r.Repository+" "+r.Commit)
|
lines = append(lines, r.Repository+" "+r.Commit)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -88,29 +87,3 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
|
|||||||
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
|
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
|
|
||||||
ctx := context.Background()
|
|
||||||
var r *rig
|
|
||||||
release, judged := make(chan struct{}), make(chan struct{})
|
|
||||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
|
||||||
if call == 1 {
|
|
||||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
|
||||||
defer close(judged)
|
|
||||||
} else {
|
|
||||||
<-release // the check asked again for the new trunk waits in the stream meanwhile
|
|
||||||
}
|
|
||||||
return Outcome{Verdict: passing()}, nil
|
|
||||||
}}
|
|
||||||
r = newRig(t, j)
|
|
||||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
|
||||||
<-judged
|
|
||||||
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
|
||||||
}
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
|
||||||
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
|
|
||||||
}
|
|
||||||
close(release)
|
|
||||||
r.ch.Wait()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -80,6 +80,9 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
|
|||||||
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
||||||
type Examiner struct {
|
type Examiner struct {
|
||||||
Asks Asks
|
Asks Asks
|
||||||
|
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
||||||
|
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
|
||||||
|
DeclineBound time.Duration
|
||||||
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
||||||
Toolchains ToolchainEntries
|
Toolchains ToolchainEntries
|
||||||
Kept Kept
|
Kept Kept
|
||||||
@@ -203,12 +206,15 @@ type state struct {
|
|||||||
nodes map[string]bool
|
nodes map[string]bool
|
||||||
clone time.Duration
|
clone time.Duration
|
||||||
slow []string
|
slow []string
|
||||||
|
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
|
||||||
|
beside map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||||
// be judged at all.
|
// be judged at all.
|
||||||
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
|
||||||
|
beside: map[string]string{}}
|
||||||
if v := st.judgeAll(ctx); v != nil {
|
if v := st.judgeAll(ctx); v != nil {
|
||||||
return st.finish(ctx, *v), nil
|
return st.finish(ctx, *v), nil
|
||||||
}
|
}
|
||||||
@@ -230,6 +236,13 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
|||||||
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
||||||
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
||||||
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||||
|
if st.c.Composition != nil && st.c.Try == nil {
|
||||||
|
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
|
||||||
|
kept := *st.c.Composition
|
||||||
|
v.Composition = &kept
|
||||||
|
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
||||||
|
return
|
||||||
|
}
|
||||||
if st.j.Compositions == nil || st.c.Try != nil {
|
if st.j.Compositions == nil || st.c.Try != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -243,6 +256,9 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
withComposition(v, comp)
|
withComposition(v, comp)
|
||||||
|
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||||
|
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
||||||
@@ -282,6 +298,9 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
out := Outcome{Verdict: v, Slowest: slow}
|
out := Outcome{Verdict: v, Slowest: slow}
|
||||||
|
if len(st.beside) > 0 {
|
||||||
|
out.Beside = st.beside
|
||||||
|
}
|
||||||
for _, t := range st.targets {
|
for _, t := range st.targets {
|
||||||
if t.judged != "" {
|
if t.judged != "" {
|
||||||
if out.Judged == nil {
|
if out.Judged == nil {
|
||||||
@@ -343,6 +362,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
|||||||
if len(beside) > 0 {
|
if len(beside) > 0 {
|
||||||
args["beside"] = beside
|
args["beside"] = beside
|
||||||
t.beside = besideLines(beside)
|
t.beside = besideLines(beside)
|
||||||
|
besideAtTrunk(st.c, beside, st.beside)
|
||||||
}
|
}
|
||||||
if p.Repository == catalogueRepository {
|
if p.Repository == catalogueRepository {
|
||||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||||
@@ -364,11 +384,25 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
||||||
|
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
|
||||||
|
const DefaultDeclineBound = 30 * time.Minute
|
||||||
|
|
||||||
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
||||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
|
||||||
|
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
|
||||||
|
// asked again of a node that has it (placed).
|
||||||
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||||
|
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
|
||||||
|
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
|
||||||
|
// is run or counted again (ADR 0552 §4).
|
||||||
|
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
||||||
p := t.pull
|
p := t.pull
|
||||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
placed := len(needs) > 0
|
||||||
|
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||||
if t.runner != "" {
|
if t.runner != "" {
|
||||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||||
}
|
}
|
||||||
@@ -380,17 +414,47 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
|||||||
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||||
}
|
}
|
||||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||||
At: st.j.now().UTC(), Check: job}
|
At: st.j.now().UTC(), Check: job, Needs: needs}
|
||||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
var step delivery.Step
|
||||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
var firstDecline time.Time
|
||||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
for {
|
||||||
}
|
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
|
||||||
step, err := st.j.Asks.Wait(ctx, a)
|
if placed {
|
||||||
if err != nil {
|
what += " for a node with " + strings.Join(needs, ", ")
|
||||||
if ctx.Err() != nil {
|
|
||||||
return fail(t.module, ctx.Err().Error())
|
|
||||||
}
|
}
|
||||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
st.c.Progress.Set(what, "")
|
||||||
|
st.c.Asked(a.ID)
|
||||||
|
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||||
|
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
step, err = st.j.Asks.Wait(ctx, a)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return fail(t.module, ctx.Err().Error())
|
||||||
|
}
|
||||||
|
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||||
|
}
|
||||||
|
d, declined := delivery.DeclinedOf(step)
|
||||||
|
if !declined {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
|
||||||
|
// node that declined looks at the stream again only after its next pull's wait.
|
||||||
|
now := st.j.now()
|
||||||
|
if firstDecline.IsZero() {
|
||||||
|
firstDecline = now
|
||||||
|
}
|
||||||
|
bound := st.j.DeclineBound
|
||||||
|
if bound <= 0 {
|
||||||
|
bound = DefaultDeclineBound
|
||||||
|
}
|
||||||
|
if now.Sub(firstDecline) > bound {
|
||||||
|
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
||||||
|
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
||||||
|
}
|
||||||
|
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
||||||
|
a.ID, a.At = buildstep.NewID(), now.UTC()
|
||||||
}
|
}
|
||||||
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
||||||
took := time.Duration(0)
|
took := time.Duration(0)
|
||||||
@@ -414,18 +478,28 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
|||||||
if st.c.Try == nil {
|
if st.c.Try == nil {
|
||||||
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||||
}
|
}
|
||||||
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
|
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
||||||
}
|
}
|
||||||
t.changed = t.fetched.Changed
|
t.changed = t.fetched.Changed
|
||||||
if !changesBuildGo(p.Repository, t.changed) {
|
if !changesBuildGo(p.Repository, t.changed) {
|
||||||
t.runner = "" // the build module used no runner: the merge does not change it
|
t.runner = "" // the build module used no runner: the merge does not change it
|
||||||
}
|
}
|
||||||
*merged = append(*merged, t.fetched)
|
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first
|
||||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
*merged = append(*merged, t.fetched)
|
||||||
return v
|
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||||
|
return v
|
||||||
|
}
|
||||||
}
|
}
|
||||||
ran := 0.0
|
ran := 0.0
|
||||||
for _, la := range answer.Levels {
|
for _, la := range answer.Levels {
|
||||||
|
if placed {
|
||||||
|
if len(la.Unmet) > 0 {
|
||||||
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||||
|
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
|
||||||
|
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
|
||||||
|
}
|
||||||
|
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
|
||||||
|
}
|
||||||
v, elapsed := st.level(ctx, t, la)
|
v, elapsed := st.level(ctx, t, la)
|
||||||
ran += elapsed
|
ran += elapsed
|
||||||
if v != nil {
|
if v != nil {
|
||||||
@@ -434,12 +508,52 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||||
|
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
||||||
|
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
|
||||||
|
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
||||||
|
var again, packages []string
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, la := range answer.Levels {
|
||||||
|
if len(again) > 0 || len(la.Unmet) > 0 {
|
||||||
|
again = append(again, la.Level)
|
||||||
|
}
|
||||||
|
for _, u := range la.Unmet {
|
||||||
|
if !seen[u.Package] {
|
||||||
|
seen[u.Package] = true
|
||||||
|
packages = append(packages, u.Package)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
||||||
|
return st.askNeeding(ctx, t, merged, again, unmet, packages)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unmetNames names the unmet packages and their needs.
|
||||||
|
func unmetNames(unmet []delivery.PackageNeeds) string {
|
||||||
|
var out []string
|
||||||
|
for _, u := range unmet {
|
||||||
|
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
|
||||||
|
}
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// unmetOf names the unmet packages of every level.
|
||||||
|
func unmetOf(levels []delivery.LevelAnswer) string {
|
||||||
|
var out []string
|
||||||
|
for _, la := range levels {
|
||||||
|
if len(la.Unmet) > 0 {
|
||||||
|
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(out, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||||
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
|
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
||||||
// verdict. It answers the level's running time (its tests' and setup's).
|
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
||||||
|
// running time (its tests' and setup's).
|
||||||
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
||||||
level := la.Level
|
level := la.Level
|
||||||
a := st.levels[level]
|
a := st.levels[level]
|
||||||
@@ -461,6 +575,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
|||||||
a.kept += k.Tests
|
a.kept += k.Tests
|
||||||
}
|
}
|
||||||
elapsed := 0.0
|
elapsed := 0.0
|
||||||
|
var failedResults []Tested
|
||||||
for _, raw := range la.Tests {
|
for _, raw := range la.Tests {
|
||||||
var res Tested
|
var res Tested
|
||||||
if err := json.Unmarshal(raw, &res); err != nil {
|
if err := json.Unmarshal(raw, &res); err != nil {
|
||||||
@@ -489,9 +604,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !res.Passed {
|
if !res.Passed {
|
||||||
return levelFailure(t, level, res), elapsed
|
failedResults = append(failedResults, res)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(failedResults) > 0 {
|
||||||
|
return levelFailure(t, level, failedResults...), elapsed
|
||||||
|
}
|
||||||
return nil, elapsed
|
return nil, elapsed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -633,23 +751,60 @@ func firstLine(s string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// failure turns a level that did not pass into a verdict naming what failed first.
|
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
|
||||||
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
|
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
|
||||||
|
const MaxFailureLine = 300
|
||||||
|
|
||||||
|
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
|
||||||
|
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
|
||||||
|
// saying how many more (issue 1921).
|
||||||
|
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
|
||||||
v := delivery.Verdict{State: delivery.VerdictFail}
|
v := delivery.Verdict{State: delivery.VerdictFail}
|
||||||
switch {
|
var all []delivery.FailedFirst
|
||||||
case len(res.Failed) > 0:
|
for _, res := range results {
|
||||||
f := res.Failed[0]
|
if len(res.Failed) == 0 {
|
||||||
name := failedName(f)
|
why := res.Why
|
||||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
if why == "" {
|
||||||
name = res.Module
|
why = "the " + level + " level did not pass"
|
||||||
|
}
|
||||||
|
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
|
for _, f := range res.Failed {
|
||||||
default:
|
name := failedName(f)
|
||||||
why := res.Why
|
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||||
if why == "" {
|
name = res.Module
|
||||||
why = "the " + level + " level did not pass"
|
}
|
||||||
|
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
|
||||||
}
|
}
|
||||||
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
|
|
||||||
}
|
}
|
||||||
|
if len(all) == 0 { // no result: a level said failed with nothing to name
|
||||||
|
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
|
||||||
|
}
|
||||||
|
v.Failed = all[0]
|
||||||
|
v.Failures = failureLines(all)
|
||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
|
||||||
|
// and then one entry saying how many more failed.
|
||||||
|
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
|
||||||
|
n := min(len(all), delivery.MaxFailures)
|
||||||
|
out := make([]delivery.FailedFirst, 0, n+1)
|
||||||
|
for _, f := range all[:n] {
|
||||||
|
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
|
||||||
|
}
|
||||||
|
if more := len(all) - n; more > 0 {
|
||||||
|
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
|
||||||
|
func clipLine(s string, n int) string {
|
||||||
|
if len(s) <= n {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
s = strings.ToValidUTF8(s[:n-3], "")
|
||||||
|
return s + "..."
|
||||||
|
}
|
||||||
|
|||||||
@@ -56,12 +56,22 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
|
|||||||
|
|
||||||
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
|
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
|
||||||
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
|
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
|
||||||
// 1): it runs every check. It answers the packages it kept now.
|
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a
|
||||||
|
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
|
||||||
|
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
|
||||||
|
// packages it kept now.
|
||||||
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
|
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
|
||||||
var kept []string
|
var kept []string
|
||||||
|
perPackage := false
|
||||||
|
for _, p := range res.Packages {
|
||||||
|
perPackage = perPackage || p.Skipped > 0
|
||||||
|
}
|
||||||
|
if res.Skipped > 0 && !perPackage {
|
||||||
|
return kept, nil // an older build module: the skip cannot be placed in a package
|
||||||
|
}
|
||||||
for _, p := range res.Packages {
|
for _, p := range res.Packages {
|
||||||
digest, ok := digests[p.Package]
|
digest, ok := digests[p.Package]
|
||||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
|
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if failedIn(res, p.Package) {
|
if failedIn(res, p.Package) {
|
||||||
|
|||||||
@@ -74,6 +74,32 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
|
||||||
|
ctx, st := context.Background(), newMemKV()
|
||||||
|
k := Kept{Store: st}
|
||||||
|
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
|
||||||
|
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
|
||||||
|
if err != nil || len(kept) != 0 {
|
||||||
|
t.Fatalf("kept %v, %v, want nothing", kept, err)
|
||||||
|
}
|
||||||
|
if keys, _ := st.Keys(ctx); len(keys) != 0 {
|
||||||
|
t.Errorf("the bucket holds %v, want none", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
|
||||||
|
ctx, st := context.Background(), newMemKV()
|
||||||
|
k := Kept{Store: st}
|
||||||
|
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
|
||||||
|
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
|
||||||
|
{Package: "m/b", Passed: true, Tests: 3},
|
||||||
|
}}
|
||||||
|
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
|
||||||
|
if err != nil || strings.Join(kept, ",") != "m/b" {
|
||||||
|
t.Fatalf("kept %v, %v, want only m/b", kept, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
|
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
|
||||||
ctx, st := context.Background(), newMemKV()
|
ctx, st := context.Background(), newMemKV()
|
||||||
k := Kept{Store: st}
|
k := Kept{Store: st}
|
||||||
|
|||||||
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
|
|||||||
return firstWordsLine(f.Words)
|
return firstWordsLine(f.Words)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
|
||||||
|
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
|
||||||
|
func failedLines(v delivery.Verdict) []string {
|
||||||
|
if len(v.Failures) == 0 {
|
||||||
|
if f := firstFailed(v); f != "" {
|
||||||
|
return []string{f}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
lines := make([]string, 0, len(v.Failures))
|
||||||
|
for _, f := range v.Failures {
|
||||||
|
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
|
||||||
|
func failedCount(v delivery.Verdict) int {
|
||||||
|
n := len(v.Failures)
|
||||||
|
if n > delivery.MaxFailures {
|
||||||
|
var more int
|
||||||
|
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
|
||||||
|
return delivery.MaxFailures + more
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
||||||
func firstWordsLine(w string) string { return firstLine(w) }
|
func firstWordsLine(w string) string { return firstLine(w) }
|
||||||
|
|
||||||
@@ -57,18 +85,20 @@ type pieces struct {
|
|||||||
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
||||||
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
|
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
|
||||||
judged string
|
judged string
|
||||||
|
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
|
||||||
|
carried string
|
||||||
}
|
}
|
||||||
|
|
||||||
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
|
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
|
||||||
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
|
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
|
||||||
|
|
||||||
func (p pieces) join(drop int) string {
|
func (p pieces) join(drop int) string {
|
||||||
parts := []string{p.head, p.commits, p.counts, p.judged}
|
parts := []string{p.head, p.commits, p.carried, p.counts, p.judged}
|
||||||
switch {
|
switch {
|
||||||
case drop < 3 && p.times != "":
|
case drop < 3 && p.times != "":
|
||||||
parts = append(parts, p.times)
|
parts = append(parts, p.times)
|
||||||
case drop == 3 && p.shortTimes != "":
|
case drop == 3 && p.shortTimes != "":
|
||||||
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
|
parts = []string{p.head, p.commits, p.carried, p.shortCounts, p.judged, p.shortTimes}
|
||||||
}
|
}
|
||||||
if p.over != "" {
|
if p.over != "" {
|
||||||
parts = append(parts, p.over)
|
parts = append(parts, p.over)
|
||||||
@@ -143,6 +173,39 @@ func buildWords(builds []BuildNote) string {
|
|||||||
return "builds: " + strings.Join(parts, ", ")
|
return "builds: " + strings.Join(parts, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
|
||||||
|
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
|
||||||
|
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
|
||||||
|
var own string
|
||||||
|
var others []string
|
||||||
|
for r, old := range from {
|
||||||
|
now, ok := v.Trunk[r]
|
||||||
|
if cs != nil && !ok {
|
||||||
|
now, ok = cs.Trunk[r]
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
now = v.Beside[r]
|
||||||
|
}
|
||||||
|
if now == "" || now == old {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.EqualFold(r, p.Repository) {
|
||||||
|
own = short12(old)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := r
|
||||||
|
if i := strings.LastIndex(r, "/"); i >= 0 {
|
||||||
|
name = r[i+1:]
|
||||||
|
}
|
||||||
|
others = append(others, name+" "+short12(old))
|
||||||
|
}
|
||||||
|
slices.Sort(others)
|
||||||
|
if own != "" {
|
||||||
|
others = append([]string{own}, others...)
|
||||||
|
}
|
||||||
|
return strings.Join(others, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
// parts makes the pieces of the line of a pull request of a case.
|
// parts makes the pieces of the line of a pull request of a case.
|
||||||
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
|
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
|
||||||
var pc pieces
|
var pc pieces
|
||||||
@@ -151,10 +214,23 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
|||||||
switch v.State {
|
switch v.State {
|
||||||
case delivery.VerdictRunning:
|
case delivery.VerdictRunning:
|
||||||
pc.head = "checking " + commits
|
pc.head = "checking " + commits
|
||||||
|
if cs != nil && cs.CarriedFrom != nil {
|
||||||
|
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
|
||||||
|
pc.head = "being carried to " + commits
|
||||||
|
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
|
||||||
|
pc.carried = "from " + w
|
||||||
|
}
|
||||||
|
}
|
||||||
case delivery.VerdictPass:
|
case delivery.VerdictPass:
|
||||||
pc.head = "pass " + commits
|
pc.head = "pass " + commits
|
||||||
|
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
|
||||||
|
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
pc.head = "failed: " + firstFailed(v)
|
pc.head = "failed: " + firstFailed(v)
|
||||||
|
if n := failedCount(v); n > 1 {
|
||||||
|
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
|
||||||
|
}
|
||||||
pc.commits = commits
|
pc.commits = commits
|
||||||
}
|
}
|
||||||
var counts []string
|
var counts []string
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
|
||||||
|
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
|
||||||
|
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
|
||||||
|
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
|
||||||
|
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
|
||||||
|
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
|
||||||
|
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
|
||||||
|
|
||||||
|
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
|
||||||
|
// judged again when the nodes change.
|
||||||
|
const LostAskTries = 3
|
||||||
|
|
||||||
|
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
|
||||||
|
func IsLostAsk(v delivery.Verdict) bool {
|
||||||
|
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
|
||||||
|
func IsNotTaken(v delivery.Verdict) bool {
|
||||||
|
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
|
||||||
|
// buildrun.HearingKey), which checker_status says (dead_letters).
|
||||||
|
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
|
||||||
|
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
|
||||||
|
// gives up (an ask no node takes).
|
||||||
|
type losingAsks struct {
|
||||||
|
*buildrun.FakeAsks
|
||||||
|
answer func(delivery.StepAsk) delivery.Step
|
||||||
|
dead *buildrun.DeadLetters
|
||||||
|
never bool
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
lose int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||||
|
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
lost := l.lose > 0
|
||||||
|
if lost {
|
||||||
|
l.lose--
|
||||||
|
}
|
||||||
|
l.mu.Unlock()
|
||||||
|
switch {
|
||||||
|
case l.never:
|
||||||
|
case lost:
|
||||||
|
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
|
||||||
|
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
|
||||||
|
g, ok := buildrun.GivenUpOf(h)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
go l.dead.Heard(g)
|
||||||
|
default:
|
||||||
|
a, err := delivery.DecodeStepAsk(data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
l.Write(l.answer(a))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func losing(b *bench, lose int) *losingAsks {
|
||||||
|
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
|
||||||
|
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
// endable is the tries state as the end tool sees it.
|
||||||
|
type endable struct{ kv *memKV }
|
||||||
|
|
||||||
|
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
|
||||||
|
v, rev, _, err := e.kv.Get(context.Background(), key)
|
||||||
|
return v, rev, err
|
||||||
|
}
|
||||||
|
func (e endable) Update(key string, value []byte, revision uint64) error {
|
||||||
|
_, err := e.kv.Update(context.Background(), key, value, revision)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
|
||||||
|
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
|
||||||
|
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
lose int
|
||||||
|
never bool
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
|
||||||
|
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
kv := newMemKV()
|
||||||
|
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
|
||||||
|
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b := newBench(t)
|
||||||
|
l := losing(b, row.lose)
|
||||||
|
l.never = row.never
|
||||||
|
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
|
||||||
|
raw, _, _, _ := kv.Get(ctx, id)
|
||||||
|
tr.Put(ctx, id, raw)
|
||||||
|
tr.Wait()
|
||||||
|
got := readTry(t, kv, id)
|
||||||
|
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
|
||||||
|
t.Fatalf("try = %+v, want error saying %q", got, row.want)
|
||||||
|
}
|
||||||
|
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
|
||||||
|
if err != nil || again == id {
|
||||||
|
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
t.Run("ended by hand", func(t *testing.T) {
|
||||||
|
kv := newMemKV()
|
||||||
|
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||||
|
b := newBench(t)
|
||||||
|
losing(b, 0).never = true
|
||||||
|
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
|
||||||
|
raw, _, _, _ := kv.Get(ctx, id)
|
||||||
|
tr.Put(ctx, id, raw)
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
|
||||||
|
t.Error("an end without a why was taken")
|
||||||
|
}
|
||||||
|
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
|
||||||
|
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
|
||||||
|
t.Fatalf("ended = %+v, %v", ended, err)
|
||||||
|
}
|
||||||
|
raw, _, _, _ = kv.Get(ctx, id)
|
||||||
|
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
|
||||||
|
tr.Wait() // returns: the try's judging stopped
|
||||||
|
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
|
||||||
|
t.Errorf("try = %+v, want the hand's end kept", got)
|
||||||
|
}
|
||||||
|
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
|
||||||
|
t.Errorf("a second end answered %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
|
||||||
|
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
|
||||||
|
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, row := range []struct {
|
||||||
|
lose int
|
||||||
|
state string
|
||||||
|
asks int
|
||||||
|
}{
|
||||||
|
{1, delivery.VerdictPass, 2},
|
||||||
|
{LostAskTries, delivery.VerdictFail, LostAskTries},
|
||||||
|
} {
|
||||||
|
b := newBench(t)
|
||||||
|
l := losing(b, row.lose)
|
||||||
|
r := newRig(t, b.ex)
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
|
r.ch.Wait()
|
||||||
|
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||||
|
if v.State != row.state || len(l.Asks()) != row.asks {
|
||||||
|
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
|
||||||
|
}
|
||||||
|
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||||
|
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -240,7 +240,12 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
|||||||
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
||||||
b.ex.Forge = forge
|
b.ex.Forge = forge
|
||||||
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
||||||
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||||
|
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
|
||||||
|
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
|
||||||
|
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
|
||||||
|
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
|
||||||
|
}
|
||||||
got := besideOf(t, b, controller)
|
got := besideOf(t, b, controller)
|
||||||
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
||||||
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
|
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
|
||||||
@@ -324,7 +329,11 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
|
|||||||
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
||||||
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||||
|
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
|
||||||
|
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
|
||||||
|
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
|
||||||
|
}
|
||||||
got := besideOf(t, b, catalogueRepository)
|
got := besideOf(t, b, catalogueRepository)
|
||||||
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
||||||
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
|
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
|
||||||
|
|||||||
@@ -75,3 +75,40 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
|
|||||||
t.Errorf("waiting = %v, want %v", st.Waiting, want)
|
t.Errorf("waiting = %v, want %v", st.Waiting, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 568 note 21: a verdict that read the node-engine at its running commit is not waiting for a trunk move while that commit
|
||||||
|
// is still the one the mesh runs, though the node-engine's trunk has moved on (ADR 0548 §6).
|
||||||
|
func TestCheckerStatusReadsTheNodeEngineAtItsRunningCommit(t *testing.T) { // 568/21
|
||||||
|
cand := candidateOf(t, group(1), pullOf(repoA, 1, head1))
|
||||||
|
ckey, _ := delivery.CandidatesKey(group(1))
|
||||||
|
vkey := keyOf(t, group(1), cand)
|
||||||
|
dc, _ := delivery.DecodeCandidate(cand)
|
||||||
|
judged, err := delivery.EncodeVerdict(delivery.Verdict{Group: group(1), Heads: dc.HeadsHash, Trunk: map[string]string{repoA: trunkA}, Beside: map[string]string{nodeEngineRepo: "run111"}, State: delivery.VerdictPass, Levels: passing().Levels})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
heads := func() (map[string]string, error) {
|
||||||
|
return map[string]string{repoA: trunkA, nodeEngineRepo: "trunk222"}, nil
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
runs runsAt
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"the running commit the verdict read", runsAt{nodeEngineModule: "run111"}, 0},
|
||||||
|
{"the running commit moved", runsAt{nodeEngineModule: "run999"}, 1},
|
||||||
|
{"no running commit known: the trunk's", nil, 1},
|
||||||
|
} {
|
||||||
|
rd := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{vkey: judged}.read(), Trunks: heads}
|
||||||
|
if row.runs != nil {
|
||||||
|
rd.Runs = row.runs
|
||||||
|
}
|
||||||
|
st, err := rd.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := len(st.Waiting[KindTrunk]); got != row.want {
|
||||||
|
t.Errorf("%s: waiting for a trunk move = %v, want %d", row.name, st.Waiting[KindTrunk], row.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||||
@@ -39,7 +40,14 @@ type Reader struct {
|
|||||||
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
||||||
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
||||||
Trunks func() (map[string]string, error)
|
Trunks func() (map[string]string, error)
|
||||||
Now func() time.Time
|
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
|
||||||
|
TurnState StateRead
|
||||||
|
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
|
||||||
|
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
|
||||||
|
Runs interface{ CommitOf(module string) string }
|
||||||
|
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
||||||
|
HearingState StateRead
|
||||||
|
Now func() time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||||
@@ -89,37 +97,46 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
|
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
|
||||||
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
|
||||||
|
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
|
||||||
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
||||||
return nil, fmt.Errorf("the verdicts state cannot be read here")
|
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
|
||||||
}
|
}
|
||||||
keys, err := r.VerdictState.Keys()
|
keys, err := r.VerdictState.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
var out []VerdictRead
|
var matching []string
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
if group != "" {
|
if group != "" {
|
||||||
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
raw, err := r.VerdictState.Get(k)
|
matching = append(matching, k)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(raw) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
vr, err := r.read(k, raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out = append(out, vr)
|
|
||||||
}
|
}
|
||||||
return out, nil
|
matching, cut = capKeys(matching, "give a group to narrow the answer")
|
||||||
|
rows := make([]VerdictRead, len(matching))
|
||||||
|
found := make([]bool, len(matching))
|
||||||
|
if err := parallel(len(matching), func(i int) error {
|
||||||
|
raw, err := r.VerdictState.Get(matching[i])
|
||||||
|
if err != nil || len(raw) == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows[i], err = r.read(matching[i], raw)
|
||||||
|
found[i] = err == nil
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
for i, row := range rows {
|
||||||
|
if found[i] {
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, cut, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
||||||
@@ -133,74 +150,95 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
|||||||
return VerdictRead{}, err
|
return VerdictRead{}, err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
for _, k := range keys {
|
keys, cut := capKeys(keys, "")
|
||||||
raw, err := r.CandidateState.Get(k)
|
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
||||||
if err != nil || len(raw) == 0 {
|
for from := 0; from < len(keys); from += ReadWorkers * 4 {
|
||||||
continue
|
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
|
||||||
}
|
cands := make([]delivery.Candidate, len(batch))
|
||||||
c, err := delivery.DecodeCandidate(raw)
|
ok := make([]bool, len(batch))
|
||||||
if err != nil {
|
_ = parallel(len(batch), func(i int) error {
|
||||||
continue
|
raw, err := r.CandidateState.Get(batch[i])
|
||||||
}
|
if err != nil || len(raw) == 0 {
|
||||||
for _, p := range c.Pulls {
|
return nil
|
||||||
if p.Repository != repository || p.Number != number {
|
}
|
||||||
|
c, err := delivery.DecodeCandidate(raw)
|
||||||
|
cands[i], ok[i] = c, err == nil
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
for i, c := range cands {
|
||||||
|
if !ok[i] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
for _, p := range c.Pulls {
|
||||||
if err != nil {
|
if p.Repository != repository || p.Number != number {
|
||||||
return VerdictRead{}, err
|
continue
|
||||||
|
}
|
||||||
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||||
|
if err != nil {
|
||||||
|
return VerdictRead{}, err
|
||||||
|
}
|
||||||
|
vraw, err := r.VerdictState.Get(vkey)
|
||||||
|
if err != nil {
|
||||||
|
return VerdictRead{}, err
|
||||||
|
}
|
||||||
|
if len(vraw) == 0 {
|
||||||
|
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||||
|
}
|
||||||
|
return r.read(vkey, vraw)
|
||||||
}
|
}
|
||||||
vraw, err := r.VerdictState.Get(vkey)
|
|
||||||
if err != nil {
|
|
||||||
return VerdictRead{}, err
|
|
||||||
}
|
|
||||||
if len(vraw) == 0 {
|
|
||||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
|
||||||
}
|
|
||||||
return r.read(vkey, vraw)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if cut != "" {
|
||||||
|
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
|
||||||
|
}
|
||||||
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
||||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
|
||||||
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
||||||
return nil, fmt.Errorf("the kept state cannot be read here")
|
return nil, "", fmt.Errorf("the kept state cannot be read here")
|
||||||
}
|
}
|
||||||
if digest != "" {
|
if digest != "" {
|
||||||
key, err := delivery.KeptKey(level, digest)
|
key, err := delivery.KeptKey(level, digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
raw, err := r.KeptState.Get(key)
|
raw, err := r.KeptState.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
return nil, fmt.Errorf("no kept verdict %s", key)
|
return nil, "", fmt.Errorf("no kept verdict %s", key)
|
||||||
}
|
}
|
||||||
return map[string]json.RawMessage{key: raw}, nil
|
return map[string]json.RawMessage{key: raw}, "", nil
|
||||||
}
|
}
|
||||||
keys, err := r.KeptState.Keys()
|
keys, err := r.KeptState.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
out := map[string]json.RawMessage{}
|
var matching []string
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
|
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
|
||||||
continue
|
matching = append(matching, k)
|
||||||
}
|
|
||||||
raw, err := r.KeptState.Get(k)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(raw) > 0 {
|
|
||||||
out[k] = raw
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
matching, cut := capKeys(matching, "give a level and a digest to read one")
|
||||||
|
values := make([]json.RawMessage, len(matching))
|
||||||
|
if err := parallel(len(matching), func(i int) (err error) {
|
||||||
|
values[i], err = r.KeptState.Get(matching[i])
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
out := map[string]json.RawMessage{}
|
||||||
|
for i, k := range matching {
|
||||||
|
if len(values[i]) > 0 {
|
||||||
|
out[k] = values[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, cut, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreviewRead is one preview as stored, with the parts of its key.
|
// PreviewRead is one preview as stored, with the parts of its key.
|
||||||
@@ -214,17 +252,18 @@ type PreviewRead struct {
|
|||||||
|
|
||||||
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
||||||
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
||||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
|
||||||
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
||||||
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
|
||||||
}
|
}
|
||||||
keys, err := r.PreviewState.Keys()
|
keys, err := r.PreviewState.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, "", err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
out := []PreviewRead{}
|
type part struct{ repo, head, trunk string }
|
||||||
var bad []string
|
parts := map[string]part{}
|
||||||
|
var matching, bad []string
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -234,15 +273,25 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, erro
|
|||||||
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
raw, err := r.PreviewState.Get(k)
|
parts[k] = part{repo, h, trunk}
|
||||||
if err != nil {
|
matching = append(matching, k)
|
||||||
return nil, nil, err
|
}
|
||||||
}
|
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
|
||||||
if len(raw) > 0 {
|
values := make([]json.RawMessage, len(matching))
|
||||||
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
if err := parallel(len(matching), func(i int) (err error) {
|
||||||
|
values[i], err = r.PreviewState.Get(matching[i])
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
return nil, nil, "", err
|
||||||
|
}
|
||||||
|
out := []PreviewRead{}
|
||||||
|
for i, k := range matching {
|
||||||
|
if len(values[i]) > 0 {
|
||||||
|
p := parts[k]
|
||||||
|
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, bad, nil
|
return out, bad, cut, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunningStep is a step in flight on a build module.
|
// RunningStep is a step in flight on a build module.
|
||||||
@@ -271,7 +320,68 @@ type Status struct {
|
|||||||
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
||||||
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
||||||
Waiting map[string][]string `json:"waiting_by_kind"`
|
Waiting map[string][]string `json:"waiting_by_kind"`
|
||||||
Notes []string `json:"notes,omitempty"`
|
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
|
||||||
|
// the group of the merge that waits. Their check is claimed when that turn goes.
|
||||||
|
HeldByTurn map[string]string `json:"held_by_turn"`
|
||||||
|
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
||||||
|
// whose ask is given up waits until its bound.
|
||||||
|
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
||||||
|
Notes []string `json:"notes,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
|
||||||
|
type DeadLettersHeard struct {
|
||||||
|
Heard bool `json:"heard"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
|
||||||
|
func (r Reader) deadLetters() DeadLettersHeard {
|
||||||
|
if r.HearingState.Get == nil {
|
||||||
|
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
|
||||||
|
}
|
||||||
|
raw, err := r.HearingState.Get(buildrun.HearingKey)
|
||||||
|
if err != nil {
|
||||||
|
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
|
||||||
|
}
|
||||||
|
a := buildrun.HearingHealth(raw)
|
||||||
|
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
|
||||||
|
}
|
||||||
|
|
||||||
|
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
|
||||||
|
func (r Reader) turns(st *Status) *Turns {
|
||||||
|
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
|
||||||
|
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
keys, err := r.TurnState.Keys()
|
||||||
|
if err != nil {
|
||||||
|
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t := &Turns{}
|
||||||
|
for _, k := range keys {
|
||||||
|
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
|
||||||
|
t.Put(k, raw, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldBy says the turn that holds a candidate's check off, or "".
|
||||||
|
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
|
||||||
|
cs := CaseOf(c, heads)
|
||||||
|
if t == nil || cs == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
repos := make([]string, 0, len(cs.Trunk))
|
||||||
|
for r := range cs.Trunk {
|
||||||
|
repos = append(repos, r)
|
||||||
|
}
|
||||||
|
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
|
||||||
|
return repo + ": the approved merge of " + w.Group + " waits on it"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// Status reads the build modules' steps and the candidates against their verdicts.
|
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||||
@@ -300,16 +410,28 @@ func (r Reader) Status() (Status, error) {
|
|||||||
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
||||||
for _, k := range keys {
|
newestFirst(keys)
|
||||||
raw, err := sr.Get(k)
|
keys, cut := capKeys(keys, "")
|
||||||
|
if cut != "" {
|
||||||
|
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
||||||
|
}
|
||||||
|
steps := make([]delivery.Step, len(keys))
|
||||||
|
isRunning := make([]bool, len(keys))
|
||||||
|
_ = parallel(len(keys), func(i int) error {
|
||||||
|
raw, err := sr.Get(keys[i])
|
||||||
if err != nil || len(raw) == 0 {
|
if err != nil || len(raw) == 0 {
|
||||||
continue
|
return nil
|
||||||
}
|
}
|
||||||
step, err := delivery.DecodeStep(raw)
|
step, err := delivery.DecodeStep(raw)
|
||||||
if err != nil || step.State != delivery.StepRunning {
|
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
for i := range keys {
|
||||||
|
if !isRunning[i] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
step := steps[i]
|
||||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||||
if !step.Started.IsZero() {
|
if !step.Started.IsZero() {
|
||||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||||
@@ -330,6 +452,8 @@ func (r Reader) Status() (Status, error) {
|
|||||||
}
|
}
|
||||||
st.Unjudged = []string{}
|
st.Unjudged = []string{}
|
||||||
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
||||||
|
st.HeldByTurn = map[string]string{}
|
||||||
|
turns := r.turns(&st)
|
||||||
var heads map[string]string
|
var heads map[string]string
|
||||||
if r.Trunks == nil {
|
if r.Trunks == nil {
|
||||||
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
||||||
@@ -344,26 +468,47 @@ func (r Reader) Status() (Status, error) {
|
|||||||
return st, err
|
return st, err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
for _, k := range keys {
|
total := len(keys)
|
||||||
raw, err := r.CandidateState.Get(k)
|
keys, cut := capKeys(keys, "")
|
||||||
|
if cut != "" {
|
||||||
|
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
|
||||||
|
}
|
||||||
|
type judged struct {
|
||||||
|
c delivery.Candidate
|
||||||
|
raw json.RawMessage
|
||||||
|
ok bool
|
||||||
|
}
|
||||||
|
rows := make([]judged, len(keys))
|
||||||
|
_ = parallel(len(keys), func(i int) error {
|
||||||
|
raw, err := r.CandidateState.Get(keys[i])
|
||||||
if err != nil || len(raw) == 0 {
|
if err != nil || len(raw) == 0 {
|
||||||
continue
|
return nil
|
||||||
}
|
}
|
||||||
c, err := delivery.DecodeCandidate(raw)
|
c, err := delivery.DecodeCandidate(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
return nil
|
||||||
}
|
}
|
||||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rows[i].c = c
|
||||||
|
if vraw, err := r.VerdictState.Get(vkey); err == nil {
|
||||||
|
rows[i].raw, rows[i].ok = vraw, true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
for _, row := range rows {
|
||||||
|
if !row.ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
raw, err = r.VerdictState.Get(vkey)
|
c, raw := row.c, row.raw
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
st.Unjudged = append(st.Unjudged, c.Group)
|
st.Unjudged = append(st.Unjudged, c.Group)
|
||||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||||
|
if why := heldBy(turns, c, heads); why != "" {
|
||||||
|
st.HeldByTurn[c.Group] = why
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
v, err := delivery.DecodeVerdict(raw)
|
v, err := delivery.DecodeVerdict(raw)
|
||||||
@@ -372,12 +517,16 @@ func (r Reader) Status() (Status, error) {
|
|||||||
case IsNodesFailure(v):
|
case IsNodesFailure(v):
|
||||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||||
case heads != nil:
|
case heads != nil:
|
||||||
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
|
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
|
||||||
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
||||||
|
if why := heldBy(turns, c, heads); why != "" {
|
||||||
|
st.HeldByTurn[c.Group] = why
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
st.DeadLetters = r.deadLetters()
|
||||||
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
||||||
return st, nil
|
return st, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,9 @@ package check
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -51,7 +53,7 @@ func readerOf(t *testing.T) (Reader, string) {
|
|||||||
|
|
||||||
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
|
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
|
||||||
rd, key := readerOf(t)
|
rd, key := readerOf(t)
|
||||||
vs, err := rd.Verdicts("")
|
vs, _, err := rd.Verdicts("")
|
||||||
if err != nil || len(vs) != 1 || vs[0].Key != key {
|
if err != nil || len(vs) != 1 || vs[0].Key != key {
|
||||||
t.Fatalf("verdicts = %+v, %v", vs, err)
|
t.Fatalf("verdicts = %+v, %v", vs, err)
|
||||||
}
|
}
|
||||||
@@ -61,7 +63,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
|
|||||||
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
|
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
|
||||||
t.Errorf("reports = %v", vs[0].Reports)
|
t.Errorf("reports = %v", vs[0].Reports)
|
||||||
}
|
}
|
||||||
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||||
t.Errorf("another group's verdicts = %+v", only)
|
t.Errorf("another group's verdicts = %+v", only)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -85,11 +87,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
|
|||||||
|
|
||||||
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
|
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
|
||||||
rd, _ := readerOf(t)
|
rd, _ := readerOf(t)
|
||||||
all, err := rd.Kept("", "")
|
all, _, err := rd.Kept("", "")
|
||||||
if err != nil || len(all) != 1 {
|
if err != nil || len(all) != 1 {
|
||||||
t.Fatalf("kept = %v, %v", all, err)
|
t.Fatalf("kept = %v, %v", all, err)
|
||||||
}
|
}
|
||||||
one, err := rd.Kept("unit", digA)
|
one, _, err := rd.Kept("unit", digA)
|
||||||
if err != nil || len(one) != 1 {
|
if err != nil || len(one) != 1 {
|
||||||
t.Fatalf("one kept = %v, %v", one, err)
|
t.Fatalf("one kept = %v, %v", one, err)
|
||||||
}
|
}
|
||||||
@@ -98,10 +100,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
|
|||||||
t.Errorf("kept value = %+v, %v", k, err)
|
t.Errorf("kept value = %+v, %v", k, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
|
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||||
t.Errorf("integration = %v", none)
|
t.Errorf("integration = %v", none)
|
||||||
}
|
}
|
||||||
if _, err := rd.Kept("unit", digB); err == nil {
|
if _, _, err := rd.Kept("unit", digB); err == nil {
|
||||||
t.Error("a digest nothing kept was answered")
|
t.Error("a digest nothing kept was answered")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -115,17 +117,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
||||||
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
||||||
got, bad, err := rd.Previews(repoA, head1[:8])
|
got, bad, _, err := rd.Previews(repoA, head1[:8])
|
||||||
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
||||||
t.Fatalf("previews = %+v, %v", got, err)
|
t.Fatalf("previews = %+v, %v", got, err)
|
||||||
}
|
}
|
||||||
if len(bad) != 1 || bad[0] != "junk" {
|
if len(bad) != 1 || bad[0] != "junk" {
|
||||||
t.Errorf("unreadable = %v", bad)
|
t.Errorf("unreadable = %v", bad)
|
||||||
}
|
}
|
||||||
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||||
t.Errorf("another repository's = %+v", got)
|
t.Errorf("another repository's = %+v", got)
|
||||||
}
|
}
|
||||||
if _, _, err := (Reader{}).Previews("", ""); err == nil {
|
if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||||
t.Error("answered with no state to read")
|
t.Error("answered with no state to read")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -142,7 +144,10 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
|||||||
CandidateState: fakeState{ckey: cand}.read(),
|
CandidateState: fakeState{ckey: cand}.read(),
|
||||||
VerdictState: fakeState{}.read(),
|
VerdictState: fakeState{}.read(),
|
||||||
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
|
||||||
|
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
|
||||||
|
`"at":"2026-10-26T13:00:00Z"}`)}.read(),
|
||||||
|
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||||
}
|
}
|
||||||
st, err := rd.Status()
|
st, err := rd.Status()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -154,9 +159,13 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
|||||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||||
t.Errorf("unjudged = %v", st.Unjudged)
|
t.Errorf("unjudged = %v", st.Unjudged)
|
||||||
}
|
}
|
||||||
|
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
|
||||||
|
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
|
||||||
|
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
|
||||||
|
}
|
||||||
// What cannot be read is said, not guessed.
|
// What cannot be read is said, not guessed.
|
||||||
st, err = Reader{}.Status()
|
st, err = Reader{}.Status()
|
||||||
if err != nil || len(st.Notes) == 0 {
|
if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard {
|
||||||
t.Errorf("empty status = %+v, %v", st, err)
|
t.Errorf("empty status = %+v, %v", st, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -232,4 +241,134 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
|||||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
||||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||||
}
|
}
|
||||||
|
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
|
||||||
|
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
hkey, _ := delivery.CandidatesKey("feat/x")
|
||||||
|
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
|
||||||
|
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
|
||||||
|
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
|
||||||
|
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||||
|
type slowBucket struct {
|
||||||
|
vals fakeState
|
||||||
|
reads, lists atomic.Int64
|
||||||
|
inFlight, maxFlight atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *slowBucket) get(key string) (json.RawMessage, error) {
|
||||||
|
b.reads.Add(1)
|
||||||
|
now := b.inFlight.Add(1)
|
||||||
|
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
b.inFlight.Add(-1)
|
||||||
|
return b.vals.read().Get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *slowBucket) read() StateRead {
|
||||||
|
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
|
||||||
|
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
|
||||||
|
func (b *slowBucket) Create(string, []byte) error { return nil }
|
||||||
|
|
||||||
|
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
|
||||||
|
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
|
||||||
|
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
|
||||||
|
kept := fakeState{}
|
||||||
|
for i := 0; i < 1800; i++ {
|
||||||
|
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
|
||||||
|
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
|
||||||
|
kept[k] = []byte(`{"v":1}`)
|
||||||
|
}
|
||||||
|
previews := fakeState{}
|
||||||
|
for i := 0; i < 1500; i++ {
|
||||||
|
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
|
||||||
|
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
|
||||||
|
previews[k] = []byte(`{"v":1}`)
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
call func(*slowBucket) (rows int, cut string, err error)
|
||||||
|
rows int
|
||||||
|
reads, lists int64
|
||||||
|
cut bool
|
||||||
|
}{
|
||||||
|
{"kept of one key", func(b *slowBucket) (int, string, error) {
|
||||||
|
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
|
||||||
|
return len(m), c, err
|
||||||
|
}, 1, 1, 0, false},
|
||||||
|
{"kept of a level", func(b *slowBucket) (int, string, error) {
|
||||||
|
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
|
||||||
|
return len(m), c, err
|
||||||
|
}, 200, 200, 1, false},
|
||||||
|
{"kept of everything", func(b *slowBucket) (int, string, error) {
|
||||||
|
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
|
||||||
|
return len(m), c, err
|
||||||
|
}, MaxReads, MaxReads, 1, true},
|
||||||
|
{"previews of a repository", func(b *slowBucket) (int, string, error) {
|
||||||
|
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
|
||||||
|
return len(p), c, err
|
||||||
|
}, 1200, 1000, 1, true},
|
||||||
|
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
|
||||||
|
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
|
||||||
|
return len(p), c, err
|
||||||
|
}, 1, 1, 1, false},
|
||||||
|
} {
|
||||||
|
b := &slowBucket{}
|
||||||
|
b.vals = kept
|
||||||
|
if strings.HasPrefix(row.name, "previews") {
|
||||||
|
b.vals = previews
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
rows, cut, err := row.call(b)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", row.name, err)
|
||||||
|
}
|
||||||
|
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
|
||||||
|
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
|
||||||
|
}
|
||||||
|
if row.reads > 20 && b.maxFlight.Load() < 2 {
|
||||||
|
t.Errorf("%s: values were read one at a time", row.name)
|
||||||
|
}
|
||||||
|
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
|
||||||
|
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
|
||||||
|
steps := fakeState{}
|
||||||
|
for i := 0; i < 1500; i++ {
|
||||||
|
id := fmt.Sprintf("mvo5%04dabcdef", i)
|
||||||
|
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
|
||||||
|
if i == 1499 {
|
||||||
|
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
|
||||||
|
}
|
||||||
|
raw, _ := delivery.EncodeStep(step)
|
||||||
|
steps["g14."+id] = raw
|
||||||
|
}
|
||||||
|
sb := &slowBucket{vals: steps}
|
||||||
|
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||||
|
Now: func() time.Time { return now }}.Status()
|
||||||
|
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
|
||||||
|
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
|
||||||
|
tries := fakeState{}
|
||||||
|
for i := 0; i < 1500; i++ {
|
||||||
|
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
|
||||||
|
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
|
||||||
|
}
|
||||||
|
tb := &slowBucket{vals: tries}
|
||||||
|
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
|
||||||
|
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package check
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
|
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
|
||||||
@@ -52,6 +54,8 @@ type GoPackage struct {
|
|||||||
Elapsed float64 `json:"elapsed"`
|
Elapsed float64 `json:"elapsed"`
|
||||||
Tests int `json:"tests"`
|
Tests int `json:"tests"`
|
||||||
NoTests bool `json:"no_tests,omitempty"`
|
NoTests bool `json:"no_tests,omitempty"`
|
||||||
|
// Skipped is how many of the package's tests skipped; an older build module names none.
|
||||||
|
Skipped int `json:"skipped,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Failure is a test that failed, or a package that failed with no test to name.
|
// Failure is a test that failed, or a package that failed with no test to name.
|
||||||
@@ -107,4 +111,6 @@ type Digested struct {
|
|||||||
WithoutTests []string `json:"without_tests,omitempty"`
|
WithoutTests []string `json:"without_tests,omitempty"`
|
||||||
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
|
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
|
||||||
JudgedBy string `json:"judged_by,omitempty"`
|
JudgedBy string `json:"judged_by,omitempty"`
|
||||||
|
// Needs are the packages whose tests need something of the node at this level (issue 890).
|
||||||
|
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
|
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
|
||||||
@@ -24,11 +25,14 @@ import (
|
|||||||
const (
|
const (
|
||||||
TryTool = "checker_try"
|
TryTool = "checker_try"
|
||||||
TryReadTool = "checker_try_read"
|
TryReadTool = "checker_try_read"
|
||||||
|
TryEndTool = "checker_try_end"
|
||||||
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
||||||
TriesState = "tries"
|
TriesState = "tries"
|
||||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||||
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
|
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
|
||||||
MaxTries = 4
|
MaxTries = 4
|
||||||
|
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
|
||||||
|
TryScan = 64
|
||||||
// TryKeep is how long an ended try's key is kept before the checker removes it.
|
// TryKeep is how long an ended try's key is kept before the checker removes it.
|
||||||
TryKeep = 7 * 24 * time.Hour
|
TryKeep = 7 * 24 * time.Hour
|
||||||
)
|
)
|
||||||
@@ -67,13 +71,18 @@ type Try struct {
|
|||||||
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
||||||
Said string `json:"said,omitempty"`
|
Said string `json:"said,omitempty"`
|
||||||
// Took is the time from its start to its end, as Go writes a duration.
|
// Took is the time from its start to its end, as Go writes a duration.
|
||||||
Took string `json:"took,omitempty"`
|
Took string `json:"took,omitempty"`
|
||||||
|
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
|
||||||
|
// (issue 1921); or what else failed first (a conflict, a refused step).
|
||||||
Failed []string `json:"failed,omitempty"`
|
Failed []string `json:"failed,omitempty"`
|
||||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||||
Machines []string `json:"machines,omitempty"`
|
Machines []string `json:"machines,omitempty"`
|
||||||
Asked time.Time `json:"asked"`
|
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||||
Started time.Time `json:"started,omitzero"`
|
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||||
Ended time.Time `json:"ended,omitzero"`
|
Asks []string `json:"asks,omitempty"`
|
||||||
|
Asked time.Time `json:"asked"`
|
||||||
|
Started time.Time `json:"started,omitzero"`
|
||||||
|
Ended time.Time `json:"ended,omitzero"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Over says whether the try came to its end.
|
// Over says whether the try came to its end.
|
||||||
@@ -155,15 +164,33 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the
|
||||||
var open []string
|
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last).
|
||||||
for _, k := range keys {
|
sort.Slice(keys, func(i, j int) bool {
|
||||||
raw, err := store.Get(k)
|
if len(keys[i]) != len(keys[j]) {
|
||||||
|
return len(keys[i]) > len(keys[j])
|
||||||
|
}
|
||||||
|
return keys[i] > keys[j]
|
||||||
|
})
|
||||||
|
if len(keys) > TryScan {
|
||||||
|
keys = keys[:TryScan]
|
||||||
|
}
|
||||||
|
tries := make([]Try, len(keys))
|
||||||
|
live := make([]bool, len(keys))
|
||||||
|
if err := parallel(len(keys), func(i int) error {
|
||||||
|
raw, err := store.Get(keys[i])
|
||||||
if err != nil || len(raw) == 0 {
|
if err != nil || len(raw) == 0 {
|
||||||
continue
|
return nil
|
||||||
}
|
}
|
||||||
cur, err := DecodeTry(raw)
|
cur, err := DecodeTry(raw)
|
||||||
if err != nil || cur.Over() {
|
tries[i], live[i] = cur, err == nil && !cur.Over()
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var open []string
|
||||||
|
for i, cur := range tries {
|
||||||
|
if !live[i] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||||
@@ -200,6 +227,52 @@ func ReadTry(store TryStore, id string) (json.RawMessage, error) {
|
|||||||
return raw, nil
|
return raw, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TryEnder is the tries state as the end tool sees it: one key with its revision, and a write at that revision.
|
||||||
|
type TryEnder interface {
|
||||||
|
Entry(key string) (json.RawMessage, uint64, error)
|
||||||
|
Update(key string, value []byte, revision uint64) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// EndTry ends a try that waits or runs as error, saying why and that it was ended by hand (issue 734): its place is
|
||||||
|
// free at once, and the next checker_try of its head asks anew. A try that ended already is refused, naming its end.
|
||||||
|
func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
|
||||||
|
if !tryKeyOK.MatchString(id) {
|
||||||
|
return Try{}, fmt.Errorf("%q is not a try's id, try-<digits>", id)
|
||||||
|
}
|
||||||
|
why = strings.TrimSpace(why)
|
||||||
|
if why == "" {
|
||||||
|
return Try{}, errors.New("say why the try is ended: the why is kept in what it says")
|
||||||
|
}
|
||||||
|
raw, rev, err := store.Entry(id)
|
||||||
|
if err != nil {
|
||||||
|
return Try{}, err
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return Try{}, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
|
||||||
|
}
|
||||||
|
t, err := DecodeTry(raw)
|
||||||
|
if err != nil {
|
||||||
|
return Try{}, err
|
||||||
|
}
|
||||||
|
if t.Over() {
|
||||||
|
return Try{}, fmt.Errorf("%s ended already, %s at %s", id, t.State, t.Ended.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
t.State, t.Ended = TryError, now.UTC()
|
||||||
|
if !t.Started.IsZero() {
|
||||||
|
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||||
|
}
|
||||||
|
t.Said = strings.TrimSpace("ended by hand (" + TryEndTool + "): " + why + " | " + t.Said)
|
||||||
|
t.Said = strings.TrimSuffix(t.Said, " |")
|
||||||
|
data, err := encodeTry(t)
|
||||||
|
if err != nil {
|
||||||
|
return Try{}, err
|
||||||
|
}
|
||||||
|
if err := store.Update(id, data, rev); err != nil {
|
||||||
|
return Try{}, fmt.Errorf("%s changed while it was ended, read it and end it again: %w", id, err)
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
// TryOptions narrow what a try runs; a pull request's check has none.
|
// TryOptions narrow what a try runs; a pull request's check has none.
|
||||||
type TryOptions struct {
|
type TryOptions struct {
|
||||||
Level string
|
Level string
|
||||||
@@ -212,12 +285,14 @@ type Tryer struct {
|
|||||||
Tries KV
|
Tries KV
|
||||||
Judge Judge
|
Judge Judge
|
||||||
Bound time.Duration
|
Bound time.Duration
|
||||||
Now func() time.Time
|
// NotTaken ends a try whose ask no build node took within it as error (issue 734); none when 0.
|
||||||
Logf func(format string, args ...any)
|
NotTaken time.Duration
|
||||||
|
Now func() time.Time
|
||||||
|
Logf func(format string, args ...any)
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
inflight map[string]bool
|
inflight map[string]context.CancelFunc
|
||||||
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,25 +320,31 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
|||||||
tr.logf("tries.%s holds what is not a try, left: %v", key, err)
|
tr.logf("tries.%s holds what is not a try, left: %v", key, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if t.Over() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tr.mu.Lock()
|
tr.mu.Lock()
|
||||||
if tr.inflight == nil {
|
if tr.inflight == nil {
|
||||||
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
|
tr.inflight, tr.seen = map[string]context.CancelFunc{}, map[string]bool{}
|
||||||
}
|
}
|
||||||
if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
|
if t.Over() {
|
||||||
|
// Ended while it ran (checker_try_end): its judging stops, and its asks are no longer waited for.
|
||||||
|
if stop := tr.inflight[key]; stop != nil {
|
||||||
|
stop()
|
||||||
|
}
|
||||||
tr.mu.Unlock()
|
tr.mu.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
|
||||||
|
tr.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
run, stop := context.WithCancel(ctx)
|
||||||
tr.seen[key] = true
|
tr.seen[key] = true
|
||||||
tr.inflight[key] = true
|
tr.inflight[key] = stop
|
||||||
tr.mu.Unlock()
|
tr.mu.Unlock()
|
||||||
tr.wg.Add(1)
|
tr.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer tr.wg.Done()
|
defer tr.wg.Done()
|
||||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
|
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock(); stop() }()
|
||||||
tr.run(ctx, key)
|
tr.run(run, key)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -320,28 +401,34 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
|||||||
Queued: t.Asked,
|
Queued: t.Asked,
|
||||||
Progress: &Progress{},
|
Progress: &Progress{},
|
||||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||||
out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
|
judging := ctx
|
||||||
|
if tr.NotTaken > 0 {
|
||||||
|
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
|
||||||
|
}
|
||||||
|
out, judged := judgeWithin(judging, tr.Judge, cs, tr.Bound)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return // left running: the checker takes it over when it starts
|
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
|
||||||
}
|
}
|
||||||
v := out.Verdict
|
v := out.Verdict
|
||||||
t.Ended = tr.now().UTC()
|
t.Ended = tr.now().UTC()
|
||||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||||
t.Levels, t.Machines = v.Levels, v.Machines
|
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||||
for _, n := range cs.Notes() {
|
for _, n := range cs.Notes() {
|
||||||
t.Said += " | " + n
|
t.Said += " | " + n
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case !judged || IsNodesFailure(v):
|
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v):
|
||||||
|
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
|
||||||
t.State = TryError
|
t.State = TryError
|
||||||
|
if IsLostAsk(v) || IsNotTaken(v) {
|
||||||
|
t.Failed = []string{v.Failed.Words}
|
||||||
|
}
|
||||||
case v.State == delivery.VerdictPass:
|
case v.State == delivery.VerdictPass:
|
||||||
t.State = TryPass
|
t.State = TryPass
|
||||||
default:
|
default:
|
||||||
t.State = TryFail
|
t.State = TryFail
|
||||||
if f := firstFailed(v); f != "" {
|
t.Failed = failedLines(v)
|
||||||
t.Failed = []string{f}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if t.Tests != "" {
|
if t.Tests != "" {
|
||||||
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
|
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
|
||||||
|
|||||||
@@ -97,6 +97,12 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
|
|||||||
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
||||||
t.Fatalf("try = %+v", got)
|
t.Fatalf("try = %+v", got)
|
||||||
}
|
}
|
||||||
|
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
|
||||||
|
if len(got.Asks) != 1 {
|
||||||
|
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
|
||||||
|
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
|
||||||
|
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
|
||||||
|
}
|
||||||
fetch := b.steps.tool("fetch")
|
fetch := b.steps.tool("fetch")
|
||||||
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
||||||
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them
|
||||||
|
// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an
|
||||||
|
// approved merge goes before a check not yet started. A bucket not there is read as empty.
|
||||||
|
type Turns struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
byRepo map[string]delivery.Turn
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that
|
||||||
|
// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing.
|
||||||
|
func (t *Turns) Put(key string, value []byte, deleted bool) bool {
|
||||||
|
repository, err := delivery.ParseTurnsKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.byRepo == nil {
|
||||||
|
t.byRepo = map[string]delivery.Turn{}
|
||||||
|
}
|
||||||
|
cur, had := t.byRepo[repository]
|
||||||
|
if !deleted {
|
||||||
|
if turn, err := delivery.DecodeTurn(value); err == nil {
|
||||||
|
t.byRepo[repository] = turn
|
||||||
|
return !had || !delivery.SameTurn(cur, turn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete(t.byRepo, repository)
|
||||||
|
return had
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that
|
||||||
|
// waits first in it; false when none does.
|
||||||
|
func (t *Turns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||||
|
if t == nil {
|
||||||
|
return "", delivery.TurnWait{}, false
|
||||||
|
}
|
||||||
|
sorted := make([]string, 0, len(repositories))
|
||||||
|
for _, r := range repositories {
|
||||||
|
sorted = append(sorted, strings.ToLower(r))
|
||||||
|
}
|
||||||
|
sort.Strings(sorted)
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
for _, r := range sorted {
|
||||||
|
turn, ok := t.byRepo[r]
|
||||||
|
if !ok || !delivery.TurnHolds(turn, heads) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, w := range turn.Waiting {
|
||||||
|
if w.Heads != heads {
|
||||||
|
return r, w, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", delivery.TurnWait{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// All is every turn held now, by repository (a copy).
|
||||||
|
func (t *Turns) All() map[string]delivery.Turn {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
out := make(map[string]delivery.Turn, len(t.byRepo))
|
||||||
|
for r, turn := range t.byRepo {
|
||||||
|
out[r] = turn
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -93,11 +93,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
|||||||
t.Errorf("module = %q", m.Module)
|
t.Errorf("module = %q", m.Module)
|
||||||
}
|
}
|
||||||
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
|
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
|
||||||
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||||
}
|
}
|
||||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||||
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
|
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
|
||||||
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||||
}
|
}
|
||||||
@@ -124,8 +124,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
|||||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||||
}
|
}
|
||||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||||
// The try's two tools (ADR 0501 §5) beside them: mesh-controller's try calls them.
|
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them.
|
||||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
||||||
],
|
],
|
||||||
"state": [
|
"state": [
|
||||||
|
{ "name": "hearing", "history": 1 },
|
||||||
{ "name": "kept", "history": 1 },
|
{ "name": "kept", "history": 1 },
|
||||||
{ "name": "owed", "history": 5 },
|
{ "name": "owed", "history": 5 },
|
||||||
{ "name": "previews", "history": 5 },
|
{ "name": "previews", "history": 5 },
|
||||||
@@ -25,6 +26,7 @@
|
|||||||
"mesh-delivery-assigner.compositions",
|
"mesh-delivery-assigner.compositions",
|
||||||
"mesh-delivery-builder.builds",
|
"mesh-delivery-builder.builds",
|
||||||
"mesh-delivery-gate.source",
|
"mesh-delivery-gate.source",
|
||||||
|
"mesh-delivery-gate.turns",
|
||||||
"mesh-delivery-indexer.candidates",
|
"mesh-delivery-indexer.candidates",
|
||||||
"mesh-delivery-indexer.manifests",
|
"mesh-delivery-indexer.manifests",
|
||||||
"mesh-delivery-indexer.modules",
|
"mesh-delivery-indexer.modules",
|
||||||
@@ -41,6 +43,7 @@
|
|||||||
"checker_pull",
|
"checker_pull",
|
||||||
"checker_status",
|
"checker_status",
|
||||||
"checker_try",
|
"checker_try",
|
||||||
|
"checker_try_end",
|
||||||
"checker_try_read",
|
"checker_try_read",
|
||||||
"checker_verdicts"
|
"checker_verdicts"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -188,18 +190,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
call := callerOn(conn, cred.User)
|
call := callerOn(conn, cred.User)
|
||||||
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
|
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
|
||||||
var turn sync.Mutex
|
var turn sync.Mutex
|
||||||
kv, err := js.KeyValue(ctx, headsBucket)
|
kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the bucket %s: %w", headsBucket, err)
|
return err
|
||||||
}
|
}
|
||||||
heads := kvHeads{kv}
|
heads := kvHeads{kv}
|
||||||
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
|
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
|
||||||
rec := &recorder{pub: js, heads: heads, logger: logger}
|
rec := &recorder{pub: js, heads: heads, logger: logger}
|
||||||
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
|
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
|
||||||
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
|
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
|
||||||
performedKV, err := js.KeyValue(ctx, performedBucket)
|
performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
|
return err
|
||||||
}
|
}
|
||||||
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
||||||
watching := make(chan error, 3)
|
watching := make(chan error, 3)
|
||||||
@@ -207,16 +209,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
bucket string
|
bucket string
|
||||||
react func(ctx context.Context, key string, rev uint64, value []byte)
|
react func(ctx context.Context, key string, rev uint64, value []byte)
|
||||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
||||||
kv, err := js.KeyValue(ctx, w.bucket)
|
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
|
||||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
|
||||||
// The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
|
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
|
||||||
// by its delivery when the gate arrives.
|
if kv == nil {
|
||||||
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
|
|
||||||
}
|
|
||||||
go func() {
|
go func() {
|
||||||
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
||||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||||
@@ -231,7 +229,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
go onReconnect(ctx, again, said, logger)
|
go onReconnect(ctx, again, said, logger)
|
||||||
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
|
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
|
// Essential: the forge's events are what the bridge carries.
|
||||||
|
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
|
||||||
}
|
}
|
||||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||||
hear(&turn, func() {
|
hear(&turn, func() {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -12,6 +13,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -66,6 +69,16 @@ func body(m map[string]any) []byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
||||||
|
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||||
|
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||||
|
var said []string
|
||||||
|
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||||
|
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||||
|
}
|
||||||
|
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||||
|
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||||
@@ -230,3 +243,8 @@ func someFiles(_, _, path, _ string) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
return nil, bridge.ErrNotFound
|
return nil, bridge.ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusingBus refuses every bucket with err (issue 1945).
|
||||||
|
type refusingBus struct{ err error }
|
||||||
|
|
||||||
|
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
|
||||||
|
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||||
|
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
|
||||||
|
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
|
||||||
|
"mesh-delivery-indexer/internal/index"
|
||||||
|
)
|
||||||
|
|
||||||
|
// asker writes a takeins value and waits for the indexer's answer to it.
|
||||||
|
type asker func(key string, value []byte) (index.TakeInAsk, error)
|
||||||
|
|
||||||
|
// Tools are the indexer's tools.
|
||||||
|
func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||||
|
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||||
|
return []stdio.Tool{
|
||||||
|
{Name: index.TakeInTool,
|
||||||
|
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
|
||||||
|
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
|
||||||
|
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
|
||||||
|
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
|
||||||
|
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
|
||||||
|
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
|
||||||
|
"indexer_module (mesh-delivery-view) reads the result.",
|
||||||
|
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||||
|
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||||
|
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
a, err := ask(key, value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return a.Result, nil
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// askTakeIn writes the takeins value at key and waits, on a watch of the key, at most bound for the indexer's answer.
|
||||||
|
func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
|
||||||
|
return func(key string, value []byte) (index.TakeInAsk, error) {
|
||||||
|
answered := make(chan index.TakeInAsk, 1)
|
||||||
|
var rev uint64
|
||||||
|
var mu sync.Mutex
|
||||||
|
take := func(revision uint64, v []byte) bool {
|
||||||
|
mu.Lock()
|
||||||
|
mine := rev
|
||||||
|
mu.Unlock()
|
||||||
|
if mine == 0 || revision <= mine {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
a, err := index.DecodeTakeIn(v)
|
||||||
|
if err != nil || a.Answered.IsZero() || a.Result == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case answered <- a:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||||
|
if c.Op == "put" {
|
||||||
|
take(c.Revision, c.Value)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return index.TakeInAsk{}, fmt.Errorf("watching %s.%s: %w", index.TakeInsState, key, err)
|
||||||
|
}
|
||||||
|
r, err := s.Put(key, json.RawMessage(value))
|
||||||
|
if err != nil {
|
||||||
|
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsBucket, err)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
rev = r
|
||||||
|
mu.Unlock()
|
||||||
|
// The watch may have heard the answer before the revision was known: read it once too.
|
||||||
|
if e, err := s.Get(key); err == nil && e != nil {
|
||||||
|
take(e.Revision, e.Value)
|
||||||
|
}
|
||||||
|
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||||
|
defer cancel()
|
||||||
|
select {
|
||||||
|
case a := <-answered:
|
||||||
|
return a, nil
|
||||||
|
case <-wait.Done():
|
||||||
|
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsBucket, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
|
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
|
||||||
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
||||||
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
||||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
|
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its
|
||||||
|
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
|
||||||
//
|
//
|
||||||
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
|
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
|
||||||
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
|
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
|
||||||
@@ -240,29 +241,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
|
||||||
kv, err := js.KeyValue(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
|
||||||
}
|
|
||||||
return kv, nil
|
|
||||||
}
|
|
||||||
candidates, err := bucket(delivery.CandidatesBucket)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
modules, err := bucket(delivery.ModulesBucket)
|
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
manifests, err := bucket(delivery.ManifestsBucket)
|
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest")
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
builds, err := bucket(delivery.BuildsBucket)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
|
||||||
|
// such a module waits for its path, every other is indexed (issue 1945).
|
||||||
|
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
|
||||||
known := index.NewBuilds()
|
known := index.NewBuilds()
|
||||||
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
|
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
|
||||||
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
|
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
|
||||||
@@ -273,12 +266,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
known.Delete(e.Key())
|
known.Delete(e.Key())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
w, err := settle(ctx, builds, each)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
errs := make(chan error, 1)
|
errs := make(chan error, 1)
|
||||||
go func() { errs <- follow(ctx, w, each) }()
|
var buildsWatch jetstream.KeyWatcher
|
||||||
|
if builds != nil {
|
||||||
|
if w, err := settle(ctx, builds, each); err != nil {
|
||||||
|
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
|
||||||
|
} else {
|
||||||
|
buildsWatch = w
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
|
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
|
||||||
account, err := process.ReadAccount(path)
|
account, err := process.ReadAccount(path)
|
||||||
@@ -291,27 +287,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
defer calls.Close()
|
defer calls.Close()
|
||||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
|
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
|
||||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
|
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call)}
|
||||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||||
sourceErrs := make(chan error, 1)
|
sourceErrs := make(chan error, 1)
|
||||||
switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
|
|
||||||
case err != nil:
|
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
|
||||||
default:
|
|
||||||
each := func(e jetstream.KeyValueEntry) {
|
each := func(e jetstream.KeyValueEntry) {
|
||||||
if opOf(e) {
|
if opOf(e) {
|
||||||
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
|
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sw, err := settle(ctx, source, each)
|
if sw, err := settle(ctx, source, each); err != nil {
|
||||||
if err != nil {
|
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again")
|
||||||
return err
|
} else {
|
||||||
|
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||||
}
|
}
|
||||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
|
||||||
}
|
}
|
||||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||||
@@ -347,6 +338,39 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||||
}
|
}
|
||||||
|
// A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records
|
||||||
|
// name it and the bucket holds none (registered.go); the records that follow are heard below.
|
||||||
|
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||||
|
logger.Printf("indexing the registered modules at start: %v", err)
|
||||||
|
}
|
||||||
|
if buildsWatch != nil {
|
||||||
|
go func() {
|
||||||
|
errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) {
|
||||||
|
each(e)
|
||||||
|
if !opOf(e) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var r delivery.BuildRecord
|
||||||
|
if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil {
|
||||||
|
logger.Printf("the registered module %s is not indexed: %v", r.Module, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
|
||||||
|
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
|
||||||
|
// merged message (index.RetryUnread).
|
||||||
|
if kv := process.OptionalBucket(ctx, js, index.TakeInsBucket, "indexer_take_in is not answered until the indexer starts again; a module the forge could not read is still read again by itself", logger.Printf); kv != nil {
|
||||||
|
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
|
||||||
|
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
|
||||||
|
process.NotWatched(logger.Printf, index.TakeInsBucket, err, "indexer_take_in is not answered until the indexer starts again")
|
||||||
|
} else {
|
||||||
|
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
|
||||||
|
}
|
||||||
|
}
|
||||||
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
|
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
|
||||||
if err := ix.Reconcile(ctx); err != nil {
|
if err := ix.Reconcile(ctx); err != nil {
|
||||||
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
|
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
|
||||||
@@ -381,7 +405,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
|
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
|
// Essential: the forge's merges are what the indexer indexes.
|
||||||
|
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
|
||||||
}
|
}
|
||||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
|
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
|
||||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||||
|
|||||||
@@ -2,11 +2,16 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-indexer/internal/index"
|
||||||
)
|
)
|
||||||
|
|
||||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
|
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
|
||||||
@@ -93,3 +98,75 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
|
|||||||
return *a.Contains, nil
|
return *a.Contains, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea gitea_get_file): arguments owner,
|
||||||
|
// repo, path and ref, answer {file: {content}}; a path it does not have fails with a 404 in its words.
|
||||||
|
const getFileTool = "gitea.gitea_get_file"
|
||||||
|
|
||||||
|
// lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles).
|
||||||
|
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||||
|
|
||||||
|
// readVia reads a repository at a commit as the forge bridge does for a merged message: the trees by gitea_commit_trees,
|
||||||
|
// then the module.json and lock files of the one directory dir by gitea_get_file (a repository can hold hundreds). A manifest the
|
||||||
|
// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out.
|
||||||
|
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||||
|
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||||
|
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||||
|
if !ok {
|
||||||
|
return index.RepoRead{}, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||||
|
}
|
||||||
|
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
|
||||||
|
if err != nil {
|
||||||
|
return index.RepoRead{}, err
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Root string `json:"root"`
|
||||||
|
Trees map[string]string `json:"trees"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return index.RepoRead{}, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||||
|
}
|
||||||
|
out := index.RepoRead{Root: a.Root, Trees: a.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}}
|
||||||
|
file := func(dir, name string) (string, error) {
|
||||||
|
p := name
|
||||||
|
if dir != "" {
|
||||||
|
p = dir + "/" + name
|
||||||
|
}
|
||||||
|
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var f struct {
|
||||||
|
File *struct {
|
||||||
|
Content *string `json:"content"`
|
||||||
|
} `json:"file"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
|
||||||
|
return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||||
|
}
|
||||||
|
return *f.File.Content, nil
|
||||||
|
}
|
||||||
|
if _, held := a.Trees[dir]; held {
|
||||||
|
m, err := file(dir, "module.json")
|
||||||
|
if err != nil {
|
||||||
|
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err)
|
||||||
|
}
|
||||||
|
out.Manifests[dir] = m
|
||||||
|
for _, name := range lockFiles {
|
||||||
|
c, err := file(dir, name)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), ": 404 ") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, name), err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(c))
|
||||||
|
if out.Locks[dir] == nil {
|
||||||
|
out.Locks[dir] = map[string]string{}
|
||||||
|
}
|
||||||
|
out.Locks[dir][name] = hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -50,3 +51,33 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
|||||||
t.Error("the trunk asked for paths")
|
t.Error("the trunk asked for paths")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||||
|
var files []string
|
||||||
|
call := func(key string, body any) (json.RawMessage, error) {
|
||||||
|
switch key {
|
||||||
|
case commitTreesTool:
|
||||||
|
return json.RawMessage(`{"root":"r0","trees":{"":"t1","modules/other":"t2"}}`), nil
|
||||||
|
case getFileTool:
|
||||||
|
files = append(files, body.(map[string]string)["path"])
|
||||||
|
switch body.(map[string]string)["path"] {
|
||||||
|
case "module.json":
|
||||||
|
return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil
|
||||||
|
case "go.sum":
|
||||||
|
return json.RawMessage(`{"file":{"content":"x"}}`), nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("Gitea API /x: 404 not found")
|
||||||
|
}
|
||||||
|
return nil, errors.New("unexpected " + key)
|
||||||
|
}
|
||||||
|
got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
|
||||||
|
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
|
||||||
|
}
|
||||||
|
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
|
||||||
|
t.Fatalf("read = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -55,11 +55,14 @@ func (w *previewWatch) ensure() error {
|
|||||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
|
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
|
||||||
return nil
|
return nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
|
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
|
||||||
|
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
|
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
w.watched = true
|
w.watched = true
|
||||||
go func() {
|
go func() {
|
||||||
|
|||||||
@@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||||
w, err := settle(ctx, kv, each)
|
w, err := settle(ctx, kv, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||||
|
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
|
return errs, nil
|
||||||
}
|
}
|
||||||
if gate.Settled() {
|
if gate.Settled() {
|
||||||
changed(ctx)
|
changed(ctx)
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"mesh-delivery-indexer/internal/index"
|
||||||
|
)
|
||||||
|
|
||||||
|
// takerIn is the part of the indexer a takeins value asks through.
|
||||||
|
type takerIn interface {
|
||||||
|
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||||
|
}
|
||||||
|
|
||||||
|
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||||
|
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||||
|
// answered already (this indexer's own write) or deleted is left.
|
||||||
|
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||||
|
if e.Operation() != jetstream.KeyValuePut {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a, err := index.DecodeTakeIn(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
logf("%s.%s holds what is not a take-in ask, left: %v", index.TakeInsState, e.Key(), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !a.Answered.IsZero() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r := ix.TakeIn(ctx, a.Module)
|
||||||
|
a.Result, a.Answered = &r, time.Now().UTC()
|
||||||
|
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||||
|
data, err := json.Marshal(a)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||||
|
logf("%s.%s: the answer could not be written: %v", index.TakeInsState, e.Key(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -73,11 +75,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
|||||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||||
return nil
|
return nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||||
|
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
|
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
w.watched[module] = true
|
w.watched[module] = true
|
||||||
go func() {
|
go func() {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -21,6 +22,7 @@ type laterBuckets struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
appears map[string]jetstream.KeyValue // buckets that exist now
|
appears map[string]jetstream.KeyValue // buckets that exist now
|
||||||
asked map[string]int
|
asked map[string]int
|
||||||
|
refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||||
@@ -30,6 +32,9 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa
|
|||||||
if kv, ok := l.appears[name]; ok {
|
if kv, ok := l.appears[name]; ok {
|
||||||
return kv, nil
|
return kv, nil
|
||||||
}
|
}
|
||||||
|
if l.refuse != nil {
|
||||||
|
return nil, l.refuse
|
||||||
|
}
|
||||||
return nil, jetstream.ErrBucketNotFound
|
return nil, jetstream.ErrBucketNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,6 +105,12 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
|
|||||||
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
|
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
|
||||||
}
|
}
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
|
// 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running.
|
||||||
|
refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")}
|
||||||
|
if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") {
|
||||||
|
t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String())
|
||||||
|
}
|
||||||
|
logs.Reset()
|
||||||
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
|
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
|
||||||
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
|
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package index
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
@@ -88,3 +90,102 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
|
|||||||
}
|
}
|
||||||
return pathOf(strings.Trim(best.Path, "/")), true
|
return pathOf(strings.Trim(best.Path, "/")), true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record
|
||||||
|
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
|
||||||
|
// built at.
|
||||||
|
type Registration struct {
|
||||||
|
Module, Repository, Path, Commit, Ref string
|
||||||
|
}
|
||||||
|
|
||||||
|
// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it
|
||||||
|
// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a
|
||||||
|
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
|
||||||
|
// built from nox-mesh, and every record of them names that ref).
|
||||||
|
func registered(r delivery.BuildRecord) bool {
|
||||||
|
return r.State == "" && r.Failed == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
|
||||||
|
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
|
||||||
|
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
|
||||||
|
// first that holds the module's manifest is the one indexed.
|
||||||
|
func (b *Builds) RegistrationsOf(module string) []Registration {
|
||||||
|
type place struct{ repository, path string }
|
||||||
|
byPlace := map[place][]delivery.BuildRecord{}
|
||||||
|
b.mu.Lock()
|
||||||
|
for _, r := range b.records {
|
||||||
|
if r.Module == module && registered(r) {
|
||||||
|
k := place{delivery.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
|
||||||
|
byPlace[k] = append(byPlace[k], r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.mu.Unlock()
|
||||||
|
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
|
||||||
|
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
|
||||||
|
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
|
||||||
|
best := map[place]delivery.BuildRecord{}
|
||||||
|
for k, recs := range byPlace {
|
||||||
|
count, latest := map[string]int{}, map[string]delivery.BuildRecord{}
|
||||||
|
for _, r := range recs {
|
||||||
|
count[r.Ref]++
|
||||||
|
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
|
||||||
|
latest[r.Ref] = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var pick string
|
||||||
|
first := true
|
||||||
|
for ref, rec := range latest {
|
||||||
|
switch {
|
||||||
|
case first:
|
||||||
|
case trunkLike[ref] != trunkLike[pick]:
|
||||||
|
if !trunkLike[ref] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
case count[ref] != count[pick]:
|
||||||
|
if count[ref] < count[pick] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
case !newer(rec, latest[pick]):
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pick, first = ref, false
|
||||||
|
}
|
||||||
|
best[k] = latest[pick]
|
||||||
|
}
|
||||||
|
recs := slices.Collect(maps.Values(best))
|
||||||
|
slices.SortFunc(recs, func(x, y delivery.BuildRecord) int {
|
||||||
|
switch {
|
||||||
|
case trunkLike[x.Ref] != trunkLike[y.Ref]:
|
||||||
|
if trunkLike[x.Ref] {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return 1
|
||||||
|
case newer(x, y):
|
||||||
|
return -1
|
||||||
|
case newer(y, x):
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
|
||||||
|
})
|
||||||
|
out := make([]Registration, 0, len(recs))
|
||||||
|
for _, r := range recs {
|
||||||
|
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Registered is the registrations of every module a record names, by module name.
|
||||||
|
func (b *Builds) Registered() []Registration {
|
||||||
|
b.mu.Lock()
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, r := range b.records {
|
||||||
|
names[r.Module] = true
|
||||||
|
}
|
||||||
|
b.mu.Unlock()
|
||||||
|
var out []Registration
|
||||||
|
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||||
|
out = append(out, b.RegistrationsOf(name)...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -53,6 +54,9 @@ type Indexer struct {
|
|||||||
// has none.
|
// has none.
|
||||||
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
|
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
|
||||||
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
|
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
|
||||||
|
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
|
||||||
|
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
|
||||||
|
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
|
||||||
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
|
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
|
||||||
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
||||||
Logf func(format string, args ...any)
|
Logf func(format string, args ...any)
|
||||||
@@ -61,6 +65,11 @@ type Indexer struct {
|
|||||||
source sourceNow
|
source sourceNow
|
||||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||||
|
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||||
|
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||||
|
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||||
|
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||||
|
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||||
recompute sync.Mutex // one Recompute at a time
|
recompute sync.Mutex // one Recompute at a time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,6 +94,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
|||||||
if err := ix.applyMerged(ctx, f); err != nil {
|
if err := ix.applyMerged(ctx, f); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
||||||
}
|
}
|
||||||
var digests map[string]string
|
var digests map[string]string
|
||||||
var held map[string]delivery.Module
|
var held map[string]delivery.Module
|
||||||
|
|||||||
@@ -25,6 +25,13 @@ import (
|
|||||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||||
|
return ix.applyManifests(ctx, f, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
|
||||||
|
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||||
|
// drops none.
|
||||||
|
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
|
||||||
digests, err := ix.keepManifests(ctx, f)
|
digests, err := ix.keepManifests(ctx, f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -84,7 +91,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
|||||||
}
|
}
|
||||||
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
|
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
|
||||||
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
|
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
|
||||||
if len(f.Manifests) > 0 && every {
|
if drop && len(f.Manifests) > 0 && every {
|
||||||
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
|
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package index
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
|
||||||
|
// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge
|
||||||
|
// followed the registration. The controller's outcome records name each registered module's repository, directory and
|
||||||
|
// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the
|
||||||
|
// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged
|
||||||
|
// message as for every other repository.
|
||||||
|
|
||||||
|
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
|
||||||
|
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
|
||||||
|
type RepoRead struct {
|
||||||
|
Root string
|
||||||
|
Trees map[string]string
|
||||||
|
Manifests map[string]string
|
||||||
|
Locks map[string]map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
|
||||||
|
// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The
|
||||||
|
// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or
|
||||||
|
// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could
|
||||||
|
// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision
|
||||||
|
// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945).
|
||||||
|
func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error {
|
||||||
|
_, err := ix.indexRegistered(ctx, r, false)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not.
|
||||||
|
type taken struct {
|
||||||
|
stored, held bool
|
||||||
|
repository, commit, dir string
|
||||||
|
why string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
|
||||||
|
if ix.Modules == nil || ix.Read == nil {
|
||||||
|
return taken{why: "the indexer reads no forge or writes no modules"}, nil
|
||||||
|
}
|
||||||
|
key, err := delivery.ModulesKey(r.Module)
|
||||||
|
if err != nil {
|
||||||
|
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
|
||||||
|
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
|
||||||
|
}
|
||||||
|
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||||
|
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
|
||||||
|
} else if found {
|
||||||
|
ix.markRead(r.Module)
|
||||||
|
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
|
||||||
|
}
|
||||||
|
repository := delivery.RepositoryName(r.Repository)
|
||||||
|
commit := r.Commit
|
||||||
|
dir := strings.Trim(r.Path, "/")
|
||||||
|
if dir == delivery.RootPath {
|
||||||
|
dir = ""
|
||||||
|
}
|
||||||
|
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
|
||||||
|
// merge removed is absent at the head and is said and left.
|
||||||
|
ix.mu.Lock()
|
||||||
|
for repo, head := range ix.source.heads {
|
||||||
|
if delivery.SameRepository(repo, r.Repository) && head != "" {
|
||||||
|
commit = head
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ix.mu.Unlock()
|
||||||
|
t := taken{repository: repository, commit: commit, dir: dir}
|
||||||
|
if repository == "" || commit == "" {
|
||||||
|
ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module)
|
||||||
|
t.why = "the controller's record names no repository or commit"
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
// One try for each repository, commit and directory: a repository with no manifest where it was registered is said once,
|
||||||
|
// not at every record of the controller's builds. A person's take-in (force) reads again.
|
||||||
|
tried := repository + "@" + commit + ":" + dir
|
||||||
|
ix.mu.Lock()
|
||||||
|
if ix.tried[tried] && !force {
|
||||||
|
ix.mu.Unlock()
|
||||||
|
t.why = "tried already at this commit and directory"
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
if ix.tried == nil {
|
||||||
|
ix.tried = map[string]bool{}
|
||||||
|
}
|
||||||
|
ix.tried[tried] = true
|
||||||
|
ix.mu.Unlock()
|
||||||
|
read, err := ix.Read(ctx, repository, commit, dir)
|
||||||
|
if err != nil {
|
||||||
|
ix.mu.Lock()
|
||||||
|
delete(ix.tried, tried) // a forge that could not say is asked again
|
||||||
|
if ix.unread == nil {
|
||||||
|
ix.unread = map[string]bool{}
|
||||||
|
}
|
||||||
|
ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread)
|
||||||
|
ix.mu.Unlock()
|
||||||
|
return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err)
|
||||||
|
}
|
||||||
|
ix.markRead(r.Module)
|
||||||
|
raw, ok := read.Manifests[dir]
|
||||||
|
if !ok || read.Trees[dir] == "" {
|
||||||
|
ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir)
|
||||||
|
t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir)
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
if name := manifestModule(raw); name != r.Module {
|
||||||
|
ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name)
|
||||||
|
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
|
||||||
|
Trees: map[string]string{dir: read.Trees[dir]},
|
||||||
|
Manifests: map[string]string{dir: raw}}
|
||||||
|
if l := read.Locks[dir]; len(l) > 0 {
|
||||||
|
f.Locks = map[string]map[string]string{dir: l}
|
||||||
|
}
|
||||||
|
if err := ix.applyManifests(ctx, f, false); err != nil {
|
||||||
|
return t, err
|
||||||
|
}
|
||||||
|
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||||
|
return t, fmt.Errorf("modules.%s: %w", key, err)
|
||||||
|
} else if !found {
|
||||||
|
t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)"
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
t.stored = true
|
||||||
|
ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit)
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// markRead takes a module out of the unread ones: the forge answered for it.
|
||||||
|
func (ix *Indexer) markRead(module string) {
|
||||||
|
ix.mu.Lock()
|
||||||
|
delete(ix.unread, module)
|
||||||
|
ix.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while
|
||||||
|
// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the
|
||||||
|
// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand
|
||||||
|
// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next.
|
||||||
|
func (ix *Indexer) RetryUnread(ctx context.Context) {
|
||||||
|
ix.mu.Lock()
|
||||||
|
names := slices.Sorted(maps.Keys(ix.unread))
|
||||||
|
ix.mu.Unlock()
|
||||||
|
for _, module := range names {
|
||||||
|
if err := ix.IndexRegisteredModule(ctx, module); err != nil {
|
||||||
|
ix.logf("the registered module %s is still not indexed: %v", module, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without
|
||||||
|
// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run
|
||||||
|
// once more when it ends; never two at once.
|
||||||
|
func (ix *Indexer) kickRetry(ctx context.Context) {
|
||||||
|
ix.mu.Lock()
|
||||||
|
none := len(ix.unread) == 0
|
||||||
|
ix.mu.Unlock()
|
||||||
|
if none {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ix.retryAgain.Store(true)
|
||||||
|
if !ix.retrying.CompareAndSwap(false, true) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ix.retryWG.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer ix.retryWG.Done()
|
||||||
|
for ix.retryAgain.Swap(false) {
|
||||||
|
ix.RetryUnread(ctx)
|
||||||
|
}
|
||||||
|
ix.retrying.Store(false)
|
||||||
|
// A kick between the last Swap and the Store found it running and returned: run for it.
|
||||||
|
if ix.retryAgain.Load() {
|
||||||
|
ix.kickRetry(ctx)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from
|
||||||
|
// where, and otherwise why not.
|
||||||
|
type TakenIn struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Stored bool `json:"stored"`
|
||||||
|
Held bool `json:"held,omitempty"`
|
||||||
|
Repository string `json:"repository,omitempty"`
|
||||||
|
Commit string `json:"commit,omitempty"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Answer string `json:"answer"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
|
||||||
|
// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the
|
||||||
|
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
|
||||||
|
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
|
||||||
|
out := TakenIn{Module: module}
|
||||||
|
if ix.Builds == nil {
|
||||||
|
out.Answer = "the indexer reads no controller's build records, so it knows no registration"
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
regs := ix.Builds.RegistrationsOf(module)
|
||||||
|
if len(regs) == 0 {
|
||||||
|
if key, err := delivery.ModulesKey(module); err == nil && ix.Modules != nil {
|
||||||
|
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
|
||||||
|
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge"
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
var whys []string
|
||||||
|
for _, r := range regs {
|
||||||
|
t, err := ix.indexRegistered(ctx, r, true)
|
||||||
|
out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
whys = append(whys, err.Error())
|
||||||
|
continue
|
||||||
|
case t.held:
|
||||||
|
out.Held, out.Answer = true, t.why
|
||||||
|
out.Repository, out.Commit, out.Path = "", "", ""
|
||||||
|
return out
|
||||||
|
case t.stored:
|
||||||
|
out.Stored = true
|
||||||
|
out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
whys = append(whys, t.why)
|
||||||
|
}
|
||||||
|
out.Answer = "not taken in: " + strings.Join(whys, "; ")
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one.
|
||||||
|
func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error {
|
||||||
|
if ix.Builds == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var first error
|
||||||
|
for _, r := range ix.Builds.RegistrationsOf(module) {
|
||||||
|
if err := ix.IndexRegistered(ctx, r); err != nil && first == nil {
|
||||||
|
first = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is
|
||||||
|
// said and the others go on; the first error is answered.
|
||||||
|
func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
|
||||||
|
if ix.Builds == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var first error
|
||||||
|
for _, r := range ix.Builds.Registered() {
|
||||||
|
if err := ix.IndexRegistered(ctx, r); err != nil {
|
||||||
|
ix.logf("the registered module %s is not indexed: %v", r.Module, err)
|
||||||
|
if first == nil {
|
||||||
|
first = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package index
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
|
||||||
|
const forge = "http://novox.internal:20000/jschoubben/snake.git"
|
||||||
|
snake := `{"module":"snake","version":"1"}`
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
read RepoRead
|
||||||
|
err error
|
||||||
|
ref string // the build record's ref
|
||||||
|
heads map[string]string // the gate source's heads
|
||||||
|
wantRead bool // the forge is read
|
||||||
|
held bool // modules.snake exists already
|
||||||
|
other bool // modules.other of the same repository exists
|
||||||
|
want bool // modules.snake is written
|
||||||
|
}{
|
||||||
|
{wantRead: true, name: "a module at the root of its repository", path: "", want: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "a root spelled as a dot", path: ".", want: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "a module in a directory", path: "modules/snake", want: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}},
|
||||||
|
{wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{name: "a module the bucket holds is left", path: "", held: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "no manifest at the path", path: "modules/x",
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "a manifest of another module", path: "",
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1"}`}}},
|
||||||
|
{wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true,
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2},
|
||||||
|
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||||
|
{wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
|
||||||
|
if tc.held {
|
||||||
|
if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.other {
|
||||||
|
if err := mods.Create(context.Background(), "other", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ix.Manifests = newMem()
|
||||||
|
var asked struct {
|
||||||
|
repo, sha, dir string
|
||||||
|
n int
|
||||||
|
}
|
||||||
|
ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) {
|
||||||
|
asked.repo, asked.sha, asked.dir = repo, sha, dir
|
||||||
|
asked.n++
|
||||||
|
return tc.read, tc.err
|
||||||
|
}
|
||||||
|
if tc.heads != nil {
|
||||||
|
ix.source = sourceNow{rev: 1, heads: tc.heads}
|
||||||
|
}
|
||||||
|
err := ix.IndexRegisteredAll(context.Background())
|
||||||
|
if (err != nil) != (tc.err != nil) {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's
|
||||||
|
if tc.heads != nil {
|
||||||
|
wantCommit = h2 // the source's head
|
||||||
|
}
|
||||||
|
if tc.wantRead && asked.sha != wantCommit {
|
||||||
|
t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit)
|
||||||
|
}
|
||||||
|
wantReads := 1
|
||||||
|
if !tc.wantRead {
|
||||||
|
wantReads = 0
|
||||||
|
}
|
||||||
|
if tc.wantRead && asked.dir != dirOf(tc.path) {
|
||||||
|
t.Errorf("the forge was asked for directory %q", asked.dir)
|
||||||
|
}
|
||||||
|
if asked.n != wantReads {
|
||||||
|
t.Errorf("forge reads = %d, want %d", asked.n, wantReads)
|
||||||
|
}
|
||||||
|
m, ok := moduleOf(t, mods, "snake")
|
||||||
|
if tc.want {
|
||||||
|
if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) {
|
||||||
|
t.Fatalf("modules.snake = %+v, found %v", m, ok)
|
||||||
|
}
|
||||||
|
if len(ix.Manifests.(*memStore).vals) != 1 {
|
||||||
|
t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals))
|
||||||
|
}
|
||||||
|
if asked.repo != "jschoubben/snake" {
|
||||||
|
t.Errorf("the forge was asked for %q", asked.repo)
|
||||||
|
}
|
||||||
|
} else if !tc.held && ok {
|
||||||
|
t.Fatalf("modules.snake written: %+v", m)
|
||||||
|
}
|
||||||
|
if _, ok := moduleOf(t, mods, "other"); ok != tc.other {
|
||||||
|
t.Errorf("modules.other held = %v, want %v", ok, tc.other)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustModule(t *testing.T, m delivery.Module) []byte {
|
||||||
|
t.Helper()
|
||||||
|
b, err := delivery.EncodeModule(withV(m))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func dirOf(p string) string {
|
||||||
|
if p == "." {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh
|
||||||
|
const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git"
|
||||||
|
ix, mods := modulesIndexer(t,
|
||||||
|
rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""),
|
||||||
|
rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""),
|
||||||
|
rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", ""))
|
||||||
|
ix.Manifests = newMem()
|
||||||
|
var asked []string
|
||||||
|
ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) {
|
||||||
|
asked = append(asked, repo+":"+dir)
|
||||||
|
if repo == "novox/Link2Pay" {
|
||||||
|
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1"}`}}, nil
|
||||||
|
}
|
||||||
|
return RepoRead{Root: "r0"}, nil
|
||||||
|
}
|
||||||
|
if err := ix.IndexRegisteredAll(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, ok := moduleOf(t, mods, "link2pay")
|
||||||
|
if !ok || m.Repository != "novox/Link2Pay" {
|
||||||
|
t.Fatalf("modules.link2pay = %+v, found %v", m, ok)
|
||||||
|
}
|
||||||
|
if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" {
|
||||||
|
t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit
|
||||||
|
// Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when
|
||||||
|
// the indexer started again. Each row's trigger must now take the module in once the forge reads it.
|
||||||
|
const snake = `{"module":"snake","version":"1"}`
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, tc := range []struct{ failure, trigger string }{
|
||||||
|
{"timeout", "the indexer starts again"},
|
||||||
|
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
|
||||||
|
{"Gitea API /repos/x/git/trees/y: 404 not found", "a new source revision"},
|
||||||
|
{"Gitea API /repos/x/git/trees/y: 404 not found", "a merged message of another repository"},
|
||||||
|
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
|
||||||
|
{"timeout", "a person's take-in"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) {
|
||||||
|
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", ""))
|
||||||
|
ix.Manifests = newMem()
|
||||||
|
ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}}
|
||||||
|
var asked []string
|
||||||
|
readable := false
|
||||||
|
ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) {
|
||||||
|
asked = append(asked, sha)
|
||||||
|
if readable {
|
||||||
|
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil
|
||||||
|
}
|
||||||
|
return RepoRead{}, errors.New(tc.failure)
|
||||||
|
}
|
||||||
|
if err := ix.IndexRegisteredAll(ctx); err == nil {
|
||||||
|
t.Errorf("no error")
|
||||||
|
}
|
||||||
|
if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 {
|
||||||
|
t.Errorf("written %v, asked %v", ok, asked)
|
||||||
|
}
|
||||||
|
if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 {
|
||||||
|
t.Errorf("not asked again (asked %v)", asked)
|
||||||
|
}
|
||||||
|
readable = true // the forge's account reads the repository now
|
||||||
|
var got TakenIn
|
||||||
|
switch tc.trigger {
|
||||||
|
case "the indexer starts again":
|
||||||
|
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "a new source revision":
|
||||||
|
if err := ix.ApplySource(ctx, 2, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ix.retryWG.Wait() // read again in the background
|
||||||
|
case "a merged message of another repository":
|
||||||
|
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ix.retryWG.Wait() // read again in the background
|
||||||
|
case "a person's take-in":
|
||||||
|
got = ix.TakeIn(ctx, "snake")
|
||||||
|
if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" {
|
||||||
|
t.Errorf("take-in answered %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 {
|
||||||
|
t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked)
|
||||||
|
}
|
||||||
|
// Taken in, it is read no more: a further event asks the forge nothing.
|
||||||
|
n := len(asked)
|
||||||
|
if err := ix.ApplySource(ctx, 3, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ix.retryWG.Wait()
|
||||||
|
if len(asked) != n {
|
||||||
|
t.Errorf("the forge was asked again after the module was taken in: %v", asked)
|
||||||
|
}
|
||||||
|
if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n {
|
||||||
|
t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked)
|
||||||
|
}
|
||||||
|
if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") {
|
||||||
|
t.Errorf("a take-in of an unregistered module answered %+v", none)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place
|
||||||
|
const repo = "http://novox.internal:20000/novox/photos.git"
|
||||||
|
b := NewBuilds()
|
||||||
|
for _, r := range [][2]any{
|
||||||
|
rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""),
|
||||||
|
rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""),
|
||||||
|
rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""),
|
||||||
|
} {
|
||||||
|
b.Put(r[0].(string), r[1].([]byte))
|
||||||
|
}
|
||||||
|
got := b.RegistrationsOf("photos")
|
||||||
|
if len(got) != 1 || got[0].Ref != "nox-mesh" {
|
||||||
|
t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got)
|
||||||
|
}
|
||||||
|
k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "")
|
||||||
|
b.Put(k, v) // a trunk's ref beats a branch of its own
|
||||||
|
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
|
||||||
|
t.Fatalf("registrations = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,6 +47,10 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
|
|||||||
if ix.Modules == nil {
|
if ix.Modules == nil {
|
||||||
return ix.RecomputeCandidates(ctx)
|
return ix.RecomputeCandidates(ctx)
|
||||||
}
|
}
|
||||||
|
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
|
||||||
|
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
|
||||||
|
// (applyManifests reads srcOf).
|
||||||
|
ix.kickRetry(ctx)
|
||||||
held, err := ix.heldModules(ctx)
|
held, err := ix.heldModules(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package index
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
|
||||||
|
// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool
|
||||||
|
// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's
|
||||||
|
// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools
|
||||||
|
// of the indexer's states stay in mesh-delivery-view.
|
||||||
|
const (
|
||||||
|
TakeInTool = "indexer_take_in"
|
||||||
|
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
|
||||||
|
TakeInsState = "takeins"
|
||||||
|
TakeInsBucket = delivery.IndexerModule + "_" + TakeInsState
|
||||||
|
)
|
||||||
|
|
||||||
|
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
|
||||||
|
// indexer read it, its answer.
|
||||||
|
type TakeInAsk struct {
|
||||||
|
V int `json:"v"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
By string `json:"by,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Result *TakenIn `json:"result,omitempty"`
|
||||||
|
Answered time.Time `json:"answered,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeTakeIn reads a takeins value.
|
||||||
|
func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||||
|
var a TakeInAsk
|
||||||
|
if err := json.Unmarshal(data, &a); err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
|
if a.V != 1 {
|
||||||
|
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||||
|
}
|
||||||
|
if a.Module == "" {
|
||||||
|
return a, errors.New("a takeins value naming no module")
|
||||||
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTakeIn is the value indexer_take_in writes for a module: its key and the value.
|
||||||
|
func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
|
||||||
|
module = strings.TrimSpace(module)
|
||||||
|
if module == "" {
|
||||||
|
return "", nil, errors.New("name the module to take in (module)")
|
||||||
|
}
|
||||||
|
key, err := delivery.ModulesKey(module)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||||
|
return key, data, err
|
||||||
|
}
|
||||||
@@ -24,18 +24,20 @@ type state struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type artifact struct {
|
type artifact struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Language string `json:"language"`
|
Language string `json:"language"`
|
||||||
System string `json:"system"`
|
System string `json:"system"`
|
||||||
From string `json:"from"`
|
From string `json:"from"`
|
||||||
Binary string `json:"binary"`
|
Binary string `json:"binary"`
|
||||||
|
Loads []string `json:"loads"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type manifest struct {
|
type manifest struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
Slug string `json:"slug"`
|
Slug string `json:"slug"`
|
||||||
|
Tools []string `json:"tools"`
|
||||||
Consumes []string `json:"consumes"`
|
Consumes []string `json:"consumes"`
|
||||||
State []state `json:"state"`
|
State []state `json:"state"`
|
||||||
Reads []string `json:"reads"`
|
Reads []string `json:"reads"`
|
||||||
@@ -78,7 +80,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
|||||||
if m.Module != "mesh-delivery-indexer" {
|
if m.Module != "mesh-delivery-indexer" {
|
||||||
t.Errorf("module = %q", m.Module)
|
t.Errorf("module = %q", m.Module)
|
||||||
}
|
}
|
||||||
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||||
}
|
}
|
||||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||||
@@ -95,14 +97,18 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
|||||||
if m.SecretsOwner != "mesh-delivery-indexer" {
|
if m.SecretsOwner != "mesh-delivery-indexer" {
|
||||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||||
}
|
}
|
||||||
// Two invokes: the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||||
// commit is in another (issue 1900).
|
// commit is in another (issue 1900).
|
||||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains"}; !reflect.DeepEqual(m.Invokes, want) {
|
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||||
}
|
}
|
||||||
// No emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
|
// 1945: one tool, indexer_take_in, served by its tools bundle through the takeins state; the process serves none.
|
||||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
|
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) {
|
||||||
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||||
|
}
|
||||||
|
// No emits, no streams, no bus, no seat (ADR 0437 §4).
|
||||||
|
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
||||||
if _, ok := top[k]; ok {
|
if _, ok := top[k]; ok {
|
||||||
t.Errorf("the manifest declares %q: the indexer has none", k)
|
t.Errorf("the manifest declares %q: the indexer has none", k)
|
||||||
}
|
}
|
||||||
@@ -134,8 +140,13 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
|
|||||||
if _, bus := p["bus"]; bus {
|
if _, bus := p["bus"]; bus {
|
||||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||||
}
|
}
|
||||||
if len(m.Build.Artifacts) != 1 {
|
if len(m.Build.Artifacts) != 2 {
|
||||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||||
|
}
|
||||||
|
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
|
||||||
|
t.Errorf("tools artifact = %+v", a) // 1945
|
||||||
|
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||||
|
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||||
}
|
}
|
||||||
a := m.Build.Artifacts[0]
|
a := m.Build.Artifacts[0]
|
||||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-indexer" || !strings.HasSuffix(a.From, a.Binary) {
|
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-indexer" || !strings.HasSuffix(a.From, a.Binary) {
|
||||||
|
|||||||
@@ -2,6 +2,9 @@
|
|||||||
"module": "mesh-delivery-indexer",
|
"module": "mesh-delivery-indexer",
|
||||||
"version": "1",
|
"version": "1",
|
||||||
"slug": "indexer",
|
"slug": "indexer",
|
||||||
|
"tools": [
|
||||||
|
"indexer_take_in"
|
||||||
|
],
|
||||||
"consumes": [
|
"consumes": [
|
||||||
"mesh-build-go.toolchain.reported",
|
"mesh-build-go.toolchain.reported",
|
||||||
"mesh-build-node.toolchain.reported",
|
"mesh-build-node.toolchain.reported",
|
||||||
@@ -11,7 +14,8 @@
|
|||||||
"state": [
|
"state": [
|
||||||
{ "name": "candidates", "history": 10 },
|
{ "name": "candidates", "history": 10 },
|
||||||
{ "name": "manifests", "history": 1 },
|
{ "name": "manifests", "history": 1 },
|
||||||
{ "name": "modules", "history": 20 }
|
{ "name": "modules", "history": 20 },
|
||||||
|
{ "name": "takeins", "history": 1 }
|
||||||
],
|
],
|
||||||
"reads": [
|
"reads": [
|
||||||
"mesh-build-go.toolchains",
|
"mesh-build-go.toolchains",
|
||||||
@@ -23,7 +27,8 @@
|
|||||||
],
|
],
|
||||||
"invokes": [
|
"invokes": [
|
||||||
"gitea.gitea_commit_trees",
|
"gitea.gitea_commit_trees",
|
||||||
"gitea.gitea_contains"
|
"gitea.gitea_contains",
|
||||||
|
"gitea.gitea_get_file"
|
||||||
],
|
],
|
||||||
"own-secrets": {
|
"own-secrets": {
|
||||||
"broker": "${dir:state}/broker"
|
"broker": "${dir:state}/broker"
|
||||||
@@ -71,6 +76,17 @@
|
|||||||
"system": "arch",
|
"system": "arch",
|
||||||
"from": "cmd/mesh-delivery-indexer",
|
"from": "cmd/mesh-delivery-indexer",
|
||||||
"binary": "mesh-delivery-indexer"
|
"binary": "mesh-delivery-indexer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tools",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/mesh-delivery-indexer-tools",
|
||||||
|
"binary": "mesh-delivery-indexer-tools",
|
||||||
|
"loads": [
|
||||||
|
"mesh-delivery-indexer-tools"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -119,12 +121,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err)
|
// Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant).
|
||||||
|
return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err)
|
||||||
}
|
}
|
||||||
|
// The answers it writes and what it asked: its own work, so each stops the start, naming its grant.
|
||||||
var kvs [2]jetstream.KeyValue
|
var kvs [2]jetstream.KeyValue
|
||||||
for i, name := range []string{delivery.AnswersBucket, askedBucket} {
|
for i, name := range []string{delivery.AnswersBucket, askedBucket} {
|
||||||
if kvs[i], err = js.KeyValue(ctx, name); err != nil {
|
if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil {
|
||||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
state := kvStore{ctx: ctx, kv: kvs[1]}
|
state := kvStore{ctx: ctx, kv: kvs[1]}
|
||||||
@@ -140,7 +144,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
|
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
|
||||||
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
||||||
failed := make(chan error, 2)
|
failed := make(chan error, 2)
|
||||||
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf)
|
||||||
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
||||||
if viewKV != nil {
|
if viewKV != nil {
|
||||||
t.View = kvGate{ctx: ctx, kv: viewKV}
|
t.View = kvGate{ctx: ctx, kv: viewKV}
|
||||||
@@ -172,15 +176,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not.
|
// teller is what an optional bucket of the teller leaves out.
|
||||||
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
const teller = "the asks and answers it holds are not carried until the teller is started again with it"
|
||||||
kv, err := js.KeyValue(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return kv
|
|
||||||
}
|
|
||||||
|
|
||||||
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
|
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
|
||||||
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
|
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
|
||||||
|
|||||||
@@ -1,12 +1,17 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
@@ -236,6 +241,16 @@ func (b *bed) askedOf(subject string) []published {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
|
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
|
||||||
|
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||||
|
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||||
|
var said []string
|
||||||
|
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||||
|
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||||
|
}
|
||||||
|
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||||
|
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
b := newBed(t)
|
b := newBed(t)
|
||||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||||
b.gate[key] = value
|
b.gate[key] = value
|
||||||
@@ -518,3 +533,8 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R
|
|||||||
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
|
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusingBus refuses every bucket with err (issue 1945).
|
||||||
|
type refusingBus struct{ err error }
|
||||||
|
|
||||||
|
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -9,13 +9,27 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
||||||
"mesh-delivery-view/internal/view"
|
"mesh-delivery-view/internal/view"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Tools are the read tools over a modules state.
|
// Tools are the read tools over a modules state, and build_usage over the build modules' states.
|
||||||
func Tools(r view.Reader) []stdio.Tool {
|
func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||||
return []stdio.Tool{
|
return []stdio.Tool{
|
||||||
|
{Name: view.BuildUsageTool,
|
||||||
|
Description: "How the build nodes are used, and whether a node is eligible to build, per node (every node holding a build module or named by a capacity): each build seat it holds (node-build-go, -node, -image) with its capacity " +
|
||||||
|
"(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " +
|
||||||
|
"running now by kind (delivery, check, recheck, try), the keys left running (past the step bound, or their module not running) and the ones " +
|
||||||
|
"their module ended for that (a restart, or past the step bound), counted apart and in no median or occupancy; " +
|
||||||
|
"the steps started and finished in the last hour and the last day, failed ones, total and median time, by kind, and how long and " +
|
||||||
|
"what share of the window the node ran its capacity of asks. Mesh-wide: the backlog of asks waiting and held on each build seat's " +
|
||||||
|
"stream, and the tries running. Read from the bus's states (have, mesh-config, the build modules' steps and pauses, the checker's tries); " +
|
||||||
|
"the backlog from the view program's reading of the streams; CPU is not on the bus and the answer names the per-node tool. Read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||||
|
Run: func(map[string]any) (any, error) { return u.Usage(), nil }},
|
||||||
{Name: view.ModuleReadTool,
|
{Name: view.ModuleReadTool,
|
||||||
Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " +
|
Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " +
|
||||||
"new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " +
|
"new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " +
|
||||||
@@ -85,7 +99,8 @@ func Tools(r view.Reader) []stdio.Tool {
|
|||||||
}},
|
}},
|
||||||
{Name: view.WantsReadTool,
|
{Name: view.WantsReadTool,
|
||||||
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
|
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
|
||||||
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
|
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given); a node and a module read that one key. " +
|
||||||
|
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
|
||||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
node, module := nodeModuleOf(args)
|
node, module := nodeModuleOf(args)
|
||||||
@@ -100,7 +115,8 @@ func Tools(r view.Reader) []stdio.Tool {
|
|||||||
}},
|
}},
|
||||||
{Name: view.DifferencesReadTool,
|
{Name: view.DifferencesReadTool,
|
||||||
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
|
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
|
||||||
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
|
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module (both: that one key). " +
|
||||||
|
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
|
||||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
node, module := nodeModuleOf(args)
|
node, module := nodeModuleOf(args)
|
||||||
@@ -108,7 +124,8 @@ func Tools(r view.Reader) []stdio.Tool {
|
|||||||
}},
|
}},
|
||||||
{Name: view.HavesReadTool,
|
{Name: view.HavesReadTool,
|
||||||
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
|
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
|
||||||
"and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
|
"and <node>.node (the node's own facts). Narrow by node and module (both: that one key). " +
|
||||||
|
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
|
||||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
node, module := nodeModuleOf(args)
|
node, module := nodeModuleOf(args)
|
||||||
@@ -127,6 +144,24 @@ func nodeModuleOf(args map[string]any) (string, string) {
|
|||||||
return strings.TrimSpace(node), strings.TrimSpace(module)
|
return strings.TrimSpace(node), strings.TrimSpace(module)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// socket is the view program's socket, as the program reads it (MESH_DELIVERY_SOCKET, else its default).
|
||||||
|
func socket() string {
|
||||||
|
if s := strings.TrimSpace(os.Getenv("MESH_DELIVERY_SOCKET")); s != "" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "/var/lib/mesh-delivery-view-cli/cli.sock"
|
||||||
|
}
|
||||||
|
|
||||||
|
// backlogOf asks the view program on its socket for the backlog it reads from the seats' streams' workers: the tool
|
||||||
|
// runtime grants a bundle states, and the stream and worker information is granted to the view program's own account
|
||||||
|
// (mesh-sdk buscompose.BuildBacklogReads).
|
||||||
|
func backlogOf(path string) func() ([]view.BacklogRow, error) {
|
||||||
|
return func() ([]view.BacklogRow, error) {
|
||||||
|
s, err := view.Ask(path)
|
||||||
|
return s.Backlog, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func read(state stdio.KeptState) view.StateRead {
|
func read(state stdio.KeptState) view.StateRead {
|
||||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||||
e, err := state.Get(key)
|
e, err := state.Get(key)
|
||||||
@@ -146,7 +181,14 @@ func read(state stdio.KeptState) view.StateRead {
|
|||||||
func main() {
|
func main() {
|
||||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
|
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
|
||||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
|
||||||
|
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
|
||||||
|
for _, lang := range delivery.AskLanguages {
|
||||||
|
module, _ := delivery.BuildModuleOf(lang)
|
||||||
|
u.Steps[lang] = read(stdio.State(module + "." + delivery.StepsState))
|
||||||
|
u.Pauses[lang] = read(stdio.State(module + "." + delivery.PausesState))
|
||||||
|
}
|
||||||
|
if err := stdio.Serve("", Tools(r, u)); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"mesh-delivery-view/internal/view"
|
"mesh-delivery-view/internal/view"
|
||||||
)
|
)
|
||||||
@@ -31,7 +33,7 @@ func (f fake) reader() view.Reader {
|
|||||||
|
|
||||||
func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) {
|
func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for _, tl := range Tools(r) {
|
for _, tl := range Tools(r, view.UsageReader{}) {
|
||||||
if tl.Name == name {
|
if tl.Name == name {
|
||||||
return tl.Run(args)
|
return tl.Run(args)
|
||||||
}
|
}
|
||||||
@@ -62,11 +64,7 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) {
|
|||||||
if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil {
|
if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil {
|
||||||
t.Errorf("stored fields are not all there: %v %v", m, err)
|
t.Errorf("stored fields are not all there: %v %v", m, err)
|
||||||
}
|
}
|
||||||
}
|
// A module with no key, and a name that is no key, are said so.
|
||||||
|
|
||||||
// A module with no key, and a name that is no key, are said so.
|
|
||||||
func TestModuleToolRefusesWhatIsNotThere(t *testing.T) {
|
|
||||||
r := fake{"a": withFP2}.reader()
|
|
||||||
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") {
|
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") {
|
||||||
t.Errorf("a missing module: %v", err)
|
t.Errorf("a missing module: %v", err)
|
||||||
}
|
}
|
||||||
@@ -164,7 +162,7 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
|
|||||||
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
|
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
|
||||||
type bigBuilds struct {
|
type bigBuilds struct {
|
||||||
fake fake
|
fake fake
|
||||||
reads *int
|
reads *atomic.Int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBig(n int) bigBuilds {
|
func newBig(n int) bigBuilds {
|
||||||
@@ -172,15 +170,15 @@ func newBig(n int) bigBuilds {
|
|||||||
for i := 0; i < n; i++ {
|
for i := 0; i < n; i++ {
|
||||||
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
|
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
|
||||||
}
|
}
|
||||||
return bigBuilds{fake: f, reads: new(int)}
|
return bigBuilds{fake: f, reads: new(atomic.Int64)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b bigBuilds) reader() view.Reader {
|
func (b bigBuilds) reader() view.Reader {
|
||||||
r := b.fake.reader()
|
r := b.fake.reader()
|
||||||
get := r.State.Get
|
get := r.State.Get
|
||||||
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
|
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { b.reads.Add(1); return get(k) },
|
||||||
GetEntry: func(k string) (json.RawMessage, uint64, error) {
|
GetEntry: func(k string) (json.RawMessage, uint64, error) {
|
||||||
*b.reads++
|
b.reads.Add(1)
|
||||||
raw, err := get(k)
|
raw, err := get(k)
|
||||||
var rev uint64
|
var rev uint64
|
||||||
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
|
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
|
||||||
@@ -198,8 +196,8 @@ func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
a := out.(view.BuildsAnswer)
|
a := out.(view.BuildsAnswer)
|
||||||
if *b.reads != 20 {
|
if b.reads.Load() != 20 {
|
||||||
t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
|
t.Errorf("read %d values for a module of 20 keys, want 20", b.reads.Load())
|
||||||
}
|
}
|
||||||
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
|
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
|
||||||
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
|
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
|
||||||
@@ -222,13 +220,13 @@ func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
a := out.(view.BuildsAnswer)
|
a := out.(view.BuildsAnswer)
|
||||||
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
|
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || b.reads.Load() > view.DefaultBuildsLimit {
|
||||||
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
|
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, b.reads.Load())
|
||||||
}
|
}
|
||||||
b = newBig(2000)
|
b = newBig(2000)
|
||||||
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
|
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
|
||||||
if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
|
if a := out.(view.BuildsAnswer); b.reads.Load() != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
|
||||||
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
|
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", b.reads.Load(), view.MaxBuildReads, a.Cut)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,6 +278,41 @@ func keptReader() view.Reader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// slowBucket is a bucket of the live shape (2026-10-25: node_haves and assigner_wants timed out at 30 s): keys over 4 nodes
|
||||||
|
// and perNode modules each, written by key(node, module), whose Get takes delay and counts the values read, the lists of
|
||||||
|
// the bucket and the most Gets in flight at once.
|
||||||
|
type slowBucket struct {
|
||||||
|
fake fake
|
||||||
|
delay time.Duration
|
||||||
|
reads, lists atomic.Int64
|
||||||
|
inFlight, maxFlight atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSlow(perNode int, delay time.Duration, key func(node string, i int) string) *slowBucket {
|
||||||
|
b := &slowBucket{fake: fake{}, delay: delay}
|
||||||
|
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
|
||||||
|
for i := 0; i < perNode; i++ {
|
||||||
|
b.fake[key(n, i)] = `{"fp":"x"}`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *slowBucket) state() view.StateRead {
|
||||||
|
f := b.fake.reader().State
|
||||||
|
return view.StateRead{
|
||||||
|
Keys: func() ([]string, error) { b.lists.Add(1); return f.Keys() },
|
||||||
|
Get: func(k string) (json.RawMessage, error) {
|
||||||
|
b.reads.Add(1)
|
||||||
|
now := b.inFlight.Add(1)
|
||||||
|
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
|
||||||
|
}
|
||||||
|
time.Sleep(b.delay)
|
||||||
|
b.inFlight.Add(-1)
|
||||||
|
return f.Get(k)
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
||||||
r := keptReader()
|
r := keptReader()
|
||||||
out, err := run(t, r, view.WantsReadTool, map[string]any{})
|
out, err := run(t, r, view.WantsReadTool, map[string]any{})
|
||||||
@@ -298,6 +331,39 @@ func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
|||||||
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
||||||
t.Errorf("module zsh = %v", got)
|
t.Errorf("module zsh = %v", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// On the live shape (4 nodes, 150 wants each, 1 ms a Get): a node and a module read that one key and list nothing;
|
||||||
|
// a node reads only its own keys; the whole answer reads its values in parallel.
|
||||||
|
slow := newSlow(150, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("want.%s.m%03d", n, i) })
|
||||||
|
r = view.Reader{WantsState: slow.state()}
|
||||||
|
for _, row := range []struct {
|
||||||
|
args map[string]any
|
||||||
|
wants int
|
||||||
|
reads, lists int64
|
||||||
|
}{
|
||||||
|
{map[string]any{"node": "ace", "module": "m007"}, 1, 1, 0},
|
||||||
|
{map[string]any{"node": "ace", "module": "gone"}, 0, 1, 0},
|
||||||
|
{map[string]any{"node": "g14"}, 150, 150, 1},
|
||||||
|
{map[string]any{"module": "m007"}, 4, 4, 1},
|
||||||
|
{map[string]any{}, 600, 600, 1},
|
||||||
|
} {
|
||||||
|
slow.reads.Store(0)
|
||||||
|
slow.lists.Store(0)
|
||||||
|
start := time.Now()
|
||||||
|
out, err := run(t, r, view.WantsReadTool, row.args)
|
||||||
|
took := time.Since(start)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%v: %v", row.args, err)
|
||||||
|
}
|
||||||
|
a := out.(view.WantsAnswer)
|
||||||
|
if len(a.Wants) != row.wants || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut {
|
||||||
|
t.Errorf("%v: %d wants, %d reads, %d lists, cut %v; want %d, %d, %d", row.args, len(a.Wants), slow.reads.Load(), slow.lists.Load(), a.Cut, row.wants, row.reads, row.lists)
|
||||||
|
}
|
||||||
|
t.Logf("assigner_wants %v: %d values in %v (one after the other: %v)", row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
|
||||||
|
}
|
||||||
|
if slow.maxFlight.Load() < 2 {
|
||||||
|
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
|
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
|
||||||
@@ -328,4 +394,39 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
|
|||||||
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
|
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
|
||||||
t.Errorf("haves of ace (the node's own key too) = %v", got)
|
t.Errorf("haves of ace (the node's own key too) = %v", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list
|
||||||
|
// nothing; the whole answer reads MaxStateReads values in parallel and says it was cut.
|
||||||
|
slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) })
|
||||||
|
r = view.Reader{HavesState: slow.state(), DifferencesState: slow.state()}
|
||||||
|
for _, row := range []struct {
|
||||||
|
tool string
|
||||||
|
args map[string]any
|
||||||
|
keys int
|
||||||
|
reads, lists int64
|
||||||
|
cut bool
|
||||||
|
}{
|
||||||
|
{view.HavesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
|
||||||
|
{view.DifferencesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
|
||||||
|
{view.HavesReadTool, map[string]any{"node": "ace"}, 300, 300, 1, false},
|
||||||
|
{view.HavesReadTool, map[string]any{"module": "m042"}, 4, 4, 1, false},
|
||||||
|
{view.HavesReadTool, map[string]any{}, view.MaxStateReads, view.MaxStateReads, 1, true},
|
||||||
|
} {
|
||||||
|
slow.reads.Store(0)
|
||||||
|
slow.lists.Store(0)
|
||||||
|
start := time.Now()
|
||||||
|
out, err := run(t, r, row.tool, row.args)
|
||||||
|
took := time.Since(start)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s %v: %v", row.tool, row.args, err)
|
||||||
|
}
|
||||||
|
a := out.(view.KeptAnswer)
|
||||||
|
if len(a.Keys) != row.keys || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut != row.cut || (a.Cut && !strings.Contains(a.Note, "narrow")) {
|
||||||
|
t.Errorf("%s %v: %d keys, %d reads, %d lists, cut %v %q; want %d, %d, %d, %v", row.tool, row.args, len(a.Keys), slow.reads.Load(), slow.lists.Load(), a.Cut, a.Note, row.keys, row.reads, row.lists, row.cut)
|
||||||
|
}
|
||||||
|
t.Logf("%s %v: %d values in %v (one after the other: %v)", row.tool, row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
|
||||||
|
}
|
||||||
|
if slow.maxFlight.Load() < 2 {
|
||||||
|
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 h1:s1WcG7Ll1OkEwwvMKEabdWvPvTiFCHghlXiY1LoIJDA=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -70,12 +72,13 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
|
values, err := r.State.values(keys)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
out := map[string]json.RawMessage{}
|
out := map[string]json.RawMessage{}
|
||||||
for _, k := range keys {
|
for i, k := range keys {
|
||||||
raw, err := r.State.Get(k)
|
raw := values[i]
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -129,13 +132,14 @@ func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string,
|
|||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
|
values, err := r.CandidatesState.values(keys)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
out := map[string]json.RawMessage{}
|
out := map[string]json.RawMessage{}
|
||||||
var unreadable []string
|
var unreadable []string
|
||||||
for _, k := range keys {
|
for i, k := range keys {
|
||||||
raw, err := r.CandidatesState.Get(k)
|
raw := values[i]
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -242,35 +246,51 @@ func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, er
|
|||||||
}
|
}
|
||||||
candidates = append(candidates, k)
|
candidates = append(candidates, k)
|
||||||
}
|
}
|
||||||
|
// The values are read in batches, each batch in parallel and taken in key order: without a module or a state a batch
|
||||||
|
// is only the rows still missing, so no more is read than the answer needs; otherwise up to readWorkers*4.
|
||||||
reads := 0
|
reads := 0
|
||||||
for _, k := range candidates {
|
walk:
|
||||||
|
for reads < len(candidates) {
|
||||||
if reads >= MaxBuildReads {
|
if reads >= MaxBuildReads {
|
||||||
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
|
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
reads++
|
size := readWorkers * 4
|
||||||
row, ok, err := r.builderRow(k)
|
if prefix == "" && state == "" {
|
||||||
if err != nil {
|
size = limit - len(out.Builds)
|
||||||
|
}
|
||||||
|
size = min(size, MaxBuildReads-reads, len(candidates)-reads)
|
||||||
|
batch := candidates[reads : reads+size]
|
||||||
|
rows := make([]BuilderRow, len(batch))
|
||||||
|
found := make([]bool, len(batch))
|
||||||
|
if err := parallel(len(batch), func(i int) error {
|
||||||
|
var err error
|
||||||
|
rows[i], found[i], err = r.builderRow(batch[i])
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
return BuildsAnswer{}, err
|
return BuildsAnswer{}, err
|
||||||
}
|
}
|
||||||
if !ok {
|
reads += len(batch)
|
||||||
continue
|
for i, row := range rows {
|
||||||
}
|
if !found[i] {
|
||||||
if state != "" {
|
|
||||||
var b struct {
|
|
||||||
State string `json:"state"`
|
|
||||||
}
|
|
||||||
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
if state != "" {
|
||||||
out.Builds = append(out.Builds, row)
|
var b struct {
|
||||||
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
State string `json:"state"`
|
||||||
if prefix == "" && len(out.Builds) >= limit {
|
}
|
||||||
if reads < len(candidates) {
|
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
|
||||||
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.Builds = append(out.Builds, row)
|
||||||
|
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
||||||
|
if prefix == "" && len(out.Builds) >= limit {
|
||||||
|
if reads-len(batch)+i+1 < len(candidates) {
|
||||||
|
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
||||||
|
}
|
||||||
|
break walk
|
||||||
}
|
}
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.SliceStable(out.Builds, func(i, j int) bool {
|
sort.SliceStable(out.Builds, func(i, j int) bool {
|
||||||
@@ -352,6 +372,9 @@ type WantsAnswer struct {
|
|||||||
Wants map[string]json.RawMessage `json:"wants"`
|
Wants map[string]json.RawMessage `json:"wants"`
|
||||||
Held map[string]json.RawMessage `json:"held"`
|
Held map[string]json.RawMessage `json:"held"`
|
||||||
Unreadable []string `json:"unreadable,omitempty"`
|
Unreadable []string `json:"unreadable,omitempty"`
|
||||||
|
// Cut says the answer stopped at MaxStateReads values, and Note how to narrow it.
|
||||||
|
Cut bool `json:"cut,omitempty"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
|
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
|
||||||
@@ -359,60 +382,158 @@ type WantsAnswer struct {
|
|||||||
type KeptAnswer struct {
|
type KeptAnswer struct {
|
||||||
Keys map[string]json.RawMessage `json:"keys"`
|
Keys map[string]json.RawMessage `json:"keys"`
|
||||||
Unreadable []string `json:"unreadable,omitempty"`
|
Unreadable []string `json:"unreadable,omitempty"`
|
||||||
|
Cut bool `json:"cut,omitempty"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
|
// readWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
||||||
if s.Keys == nil || s.Get == nil {
|
// runtime to the bus (the runtime opens the bucket and gets the key: two bus requests), so reading a bucket of a few
|
||||||
return nil, fmt.Errorf("%s cannot be read here", what)
|
// hundred keys one after the other outlasted the 30 s a tool call is given on 2026-10-25 (node_haves, assigner_wants).
|
||||||
}
|
const readWorkers = 16
|
||||||
keys, err := s.Keys()
|
|
||||||
if err != nil {
|
// MaxStateReads bounds the values one call of assigner_wants, assigner_differences or node_haves reads; past it the
|
||||||
return nil, err
|
// answer says it was cut and how to narrow it.
|
||||||
|
const MaxStateReads = 1000
|
||||||
|
|
||||||
|
// parallel runs read(i) for i in [0, n), at most readWorkers at a time, and answers the first error; once one failed no
|
||||||
|
// further read starts.
|
||||||
|
func parallel(n int, read func(i int) error) error {
|
||||||
|
var (
|
||||||
|
next atomic.Int64
|
||||||
|
failed atomic.Bool
|
||||||
|
once sync.Once
|
||||||
|
first error
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
workers := min(readWorkers, n)
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for !failed.Load() {
|
||||||
|
i := int(next.Add(1) - 1)
|
||||||
|
if i >= n {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := read(i); err != nil {
|
||||||
|
once.Do(func() { first = err })
|
||||||
|
failed.Store(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
wg.Wait()
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
// values reads the stored value of each key, in parallel, in the order of keys (nil for a key with none).
|
||||||
|
func (s StateRead) values(keys []string) ([]json.RawMessage, error) {
|
||||||
|
out := make([]json.RawMessage, len(keys))
|
||||||
|
err := parallel(len(keys), func(i int) error {
|
||||||
|
raw, err := s.Get(keys[i])
|
||||||
|
out[i] = raw
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// read reads the values of keys (sorted first) into a map, at most MaxStateReads of them; cut says it stopped there.
|
||||||
|
func (s StateRead) read(keys []string) (map[string]json.RawMessage, bool, error) {
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
|
cut := len(keys) > MaxStateReads
|
||||||
|
if cut {
|
||||||
|
keys = keys[:MaxStateReads]
|
||||||
|
}
|
||||||
|
values, err := s.values(keys)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
out := map[string]json.RawMessage{}
|
out := map[string]json.RawMessage{}
|
||||||
for _, k := range keys {
|
for i, k := range keys {
|
||||||
raw, err := s.Get(k)
|
if len(values[i]) > 0 {
|
||||||
if err != nil {
|
out[k] = values[i]
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(raw) > 0 {
|
|
||||||
out[k] = raw
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, cut, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cutNote(cut bool, n int) string {
|
||||||
|
if !cut {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("read the first %d of %d matching keys in key order and stopped; give a node and a module to narrow the answer", MaxStateReads, n)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
|
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
|
||||||
// matches).
|
// matches).
|
||||||
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
|
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
|
||||||
all, err := r.WantsState.all("the assigner's wants")
|
s := r.WantsState
|
||||||
|
if s.Get == nil {
|
||||||
|
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
|
||||||
|
}
|
||||||
|
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
|
||||||
|
if node != "" && module != "" {
|
||||||
|
// One key, read directly: no list of the bucket (a hold has no node, so none is shown).
|
||||||
|
key, err := delivery.WantKey(node, module)
|
||||||
|
if module == "node" {
|
||||||
|
key, err = delivery.WantNodeKey(node)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return WantsAnswer{}, err
|
||||||
|
}
|
||||||
|
raw, err := s.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return WantsAnswer{}, err
|
||||||
|
}
|
||||||
|
if len(raw) > 0 {
|
||||||
|
out.Wants[key] = raw
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if s.Keys == nil {
|
||||||
|
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
|
||||||
|
}
|
||||||
|
keys, err := s.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return WantsAnswer{}, err
|
return WantsAnswer{}, err
|
||||||
}
|
}
|
||||||
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
|
// Narrowed by the key before any value is read.
|
||||||
for k, raw := range all {
|
var matching []string
|
||||||
|
for _, k := range keys {
|
||||||
if n, m, err := delivery.ParseWantKey(k); err == nil {
|
if n, m, err := delivery.ParseWantKey(k); err == nil {
|
||||||
if (node == "" || n == node) && (module == "" || m == module) {
|
if (node == "" || n == node) && (module == "" || m == module) {
|
||||||
out.Wants[k] = raw
|
matching = append(matching, k)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
|
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
|
||||||
if n, err := delivery.ParseWantNodeKey(k); err == nil {
|
if n, err := delivery.ParseWantNodeKey(k); err == nil {
|
||||||
if (node == "" || n == node) && (module == "" || module == "node") {
|
if (node == "" || n == node) && (module == "" || module == "node") {
|
||||||
out.Wants[k] = raw
|
matching = append(matching, k)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if m, _, err := delivery.ParseHeldKey(k); err == nil {
|
if m, _, err := delivery.ParseHeldKey(k); err == nil {
|
||||||
if node == "" && (module == "" || m == module) {
|
if node == "" && (module == "" || m == module) {
|
||||||
out.Held[k] = raw
|
matching = append(matching, k)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out.Unreadable = append(out.Unreadable, k)
|
out.Unreadable = append(out.Unreadable, k)
|
||||||
}
|
}
|
||||||
|
sort.Strings(out.Unreadable)
|
||||||
|
values, cut, err := s.read(matching)
|
||||||
|
if err != nil {
|
||||||
|
return WantsAnswer{}, err
|
||||||
|
}
|
||||||
|
out.Cut, out.Note = cut, cutNote(cut, len(matching))
|
||||||
|
for k, raw := range values {
|
||||||
|
if _, _, err := delivery.ParseHeldKey(k); err == nil {
|
||||||
|
out.Held[k] = raw
|
||||||
|
} else {
|
||||||
|
out.Wants[k] = raw
|
||||||
|
}
|
||||||
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,21 +557,50 @@ func (r Reader) Want(node, module string) (json.RawMessage, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
|
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
|
||||||
all, err := s.all(what)
|
if s.Get == nil {
|
||||||
|
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
|
||||||
|
}
|
||||||
|
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
|
||||||
|
if node != "" && module != "" {
|
||||||
|
// One key, read directly: no list of the bucket.
|
||||||
|
key, err := delivery.HaveKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return KeptAnswer{}, err
|
||||||
|
}
|
||||||
|
raw, err := s.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return KeptAnswer{}, err
|
||||||
|
}
|
||||||
|
if len(raw) > 0 {
|
||||||
|
out.Keys[key] = raw
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if s.Keys == nil {
|
||||||
|
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
|
||||||
|
}
|
||||||
|
keys, err := s.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return KeptAnswer{}, err
|
return KeptAnswer{}, err
|
||||||
}
|
}
|
||||||
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
|
// Narrowed by the key before any value is read.
|
||||||
for k, raw := range all {
|
var matching []string
|
||||||
|
for _, k := range keys {
|
||||||
n, m, isNode, err := delivery.ParseHaveKey(k)
|
n, m, isNode, err := delivery.ParseHaveKey(k)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out.Unreadable = append(out.Unreadable, k)
|
out.Unreadable = append(out.Unreadable, k)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
|
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
|
||||||
out.Keys[k] = raw
|
matching = append(matching, k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sort.Strings(out.Unreadable)
|
||||||
|
values, cut, err := s.read(matching)
|
||||||
|
if err != nil {
|
||||||
|
return KeptAnswer{}, err
|
||||||
|
}
|
||||||
|
out.Keys, out.Cut, out.Note = values, cut, cutNote(cut, len(matching))
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user