Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9132b6a296 | ||
|
|
bcb5aafdbe | ||
|
|
3e8bd9b0de | ||
|
|
b842b9422d | ||
|
|
9db15c7cf4 | ||
|
|
ba47b47e24 | ||
|
|
b566563dc5 | ||
|
|
2a856e2a76 | ||
|
|
ce7d60cb6e | ||
|
|
26ba7404bf | ||
|
|
b4af9cdb7a | ||
|
|
db53ec975f | ||
|
|
1ff5049d03 | ||
|
|
41f25d3129 | ||
|
|
4628ba26b4 | ||
|
|
20027e53ff | ||
|
|
d4a054bfed | ||
|
|
c623e8d235 | ||
|
|
998fe3a515 | ||
|
|
73be3df28c | ||
|
|
eb65535f45 | ||
|
|
fa6cdbbe61 | ||
|
|
462ec9cfb1 | ||
|
|
9950689a11 | ||
|
|
71a6a5bf77 | ||
|
|
22d4cbd144 | ||
|
|
7b618b2317 | ||
|
|
d8477c71b1 | ||
|
|
45fae9da8a | ||
|
|
2f742fd3fe | ||
|
|
82180bffd2 | ||
|
|
ac82187b85 | ||
|
|
8d38c6bac9 | ||
|
|
9afef800c0 | ||
|
|
eaf27f36af | ||
|
|
4efe6cfb26 | ||
|
|
3e7f852472 |
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
@@ -125,8 +126,8 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
return done, stopped
|
||||
}
|
||||
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
|
||||
// having opened every bucket it names.
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
|
||||
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
|
||||
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
done := b.start(t)
|
||||
@@ -150,9 +151,21 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
}
|
||||
break
|
||||
}
|
||||
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") ||
|
||||
!said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") {
|
||||
t.Fatalf("the assigner said %+v, want its four states, each readable", said)
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != len(m.State) {
|
||||
t.Fatalf("the assigner said %+v, want the %d states its manifest declares", said, len(m.State))
|
||||
}
|
||||
for _, s := range m.State {
|
||||
if !said.Readable(s.Name) {
|
||||
t.Errorf("the assigner said %+v: its state %s is not readable", said, s.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +182,42 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
t.Errorf("without %s the assigner started", missing)
|
||||
}
|
||||
}
|
||||
// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
|
||||
// ADR 0201) a pass opens it, so a put is taken further without a restart.
|
||||
was := PutsReopenEvery
|
||||
PutsReopenEvery = 0
|
||||
defer func() { PutsReopenEvery = was }()
|
||||
b := startBed(t, delivery.AssignerPutsBucket)
|
||||
done := b.start(t)
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PutKey("n1", "web")
|
||||
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
|
||||
raw, _ := delivery.EncodePut(asked)
|
||||
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
|
||||
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := puts.Get(b.ctx, pkey); err == nil {
|
||||
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the put was not taken further once its bucket was made")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// putJSON writes a value into one of the bed's buckets.
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -29,6 +30,15 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||
if err := serveTools(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
@@ -59,29 +69,37 @@ var readBuckets = []string{
|
||||
const goBuildModule = "mesh-build-go"
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller.
|
||||
// and the teller, its puts by whoever follows one (ADR 0546).
|
||||
var keptStates = []announce.State{
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
{Name: "puts", Readable: true},
|
||||
{Name: "wants", Readable: true},
|
||||
}
|
||||
|
||||
// buckets is every bucket the assigner opened, by name.
|
||||
type buckets map[string]jetstream.KeyValue
|
||||
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
||||
// without it would write wants from half the mesh.
|
||||
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
|
||||
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
||||
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
out := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
out[name] = kv
|
||||
}
|
||||
@@ -118,7 +136,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvs, err := open(ctx, js)
|
||||
kvs, err := open(ctx, js, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -133,9 +151,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
}
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
|
||||
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
@@ -156,7 +179,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -165,13 +188,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
default:
|
||||
}
|
||||
}
|
||||
errs := make(chan error, len(watched)+1)
|
||||
settled := make(chan struct{}, len(watched)+1)
|
||||
errs := make(chan error, len(watched)+2)
|
||||
settled := make(chan struct{}, len(watched)+2)
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
||||
}
|
||||
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
|
||||
optional := name == delivery.AssignerPutsBucket
|
||||
go func() {
|
||||
defer kw.Stop()
|
||||
for {
|
||||
@@ -181,11 +206,17 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
return
|
||||
case e, ok := <-kw.Updates():
|
||||
if !ok {
|
||||
if optional {
|
||||
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
|
||||
return
|
||||
}
|
||||
errs <- fmt.Errorf("the watch of %s ended", name)
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
settled <- struct{}{}
|
||||
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
|
||||
settled <- struct{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
var value []byte
|
||||
@@ -199,6 +230,39 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
|
||||
// stops the puts, never the wants.
|
||||
if kvs[delivery.AssignerPutsBucket] != nil {
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
}
|
||||
}
|
||||
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
|
||||
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
|
||||
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
|
||||
// further, without a restart.
|
||||
var triedPuts time.Time
|
||||
reopenPuts := func(p *passer) {
|
||||
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
|
||||
return
|
||||
}
|
||||
triedPuts = time.Now()
|
||||
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket] = kv
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
return
|
||||
}
|
||||
if bus != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
for _, name := range watched {
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
@@ -222,6 +286,9 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -231,11 +298,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
return err
|
||||
case <-settled:
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||
var PutsReopenEvery = time.Minute
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
|
||||
@@ -2,15 +2,23 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
@@ -48,6 +56,53 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
}
|
||||
}
|
||||
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
|
||||
// keep the first pass from being made: the wants are written all the same.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
wrote := make(chan string, 8)
|
||||
kvs := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
t.Errorf("the first pass wrote %s", key)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
@@ -57,3 +112,61 @@ func TestNoCredential(t *testing.T) {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
// closed; a put on it is said on wrote.
|
||||
type watchKV struct {
|
||||
*memKV
|
||||
entries []kvEntry
|
||||
closed bool
|
||||
wrote chan string
|
||||
}
|
||||
|
||||
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
|
||||
if w.closed {
|
||||
close(ch)
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
for _, e := range w.entries {
|
||||
ch <- e
|
||||
}
|
||||
ch <- nil
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
|
||||
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if w.wrote != nil {
|
||||
w.wrote <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
|
||||
|
||||
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
|
||||
func (k keyWatcher) Stop() error { return nil }
|
||||
|
||||
type kvEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -33,6 +34,11 @@ type passer struct {
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
// first pass after a start, which clears what a previous process may have raised (startClears).
|
||||
neither map[string]neitherOne
|
||||
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
||||
acts *putActs
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
@@ -45,6 +51,7 @@ func (p *passer) clock() time.Time {
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.compositions(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
@@ -90,6 +97,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
puts := p.putsByModule()
|
||||
names := map[string]bool{}
|
||||
for name := range trunk {
|
||||
names[name] = true
|
||||
@@ -108,7 +116,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(),
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
@@ -144,6 +152,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
gatherNeither(neither, name, out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
@@ -182,6 +191,42 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
|
||||
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
|
||||
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
|
||||
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
|
||||
if p.noWant == nil {
|
||||
p.noWant = map[string]string{}
|
||||
}
|
||||
assigned := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
assigned[node] = true
|
||||
key := node + "." + module
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
case reason != "" && reason != last:
|
||||
p.logf("%s on %s gets no want: %s", module, node, reason)
|
||||
p.noWant[key] = reason
|
||||
case reason == "" && said && written && !w.Removed:
|
||||
p.logf("%s on %s has a want", module, node)
|
||||
delete(p.noWant, key)
|
||||
}
|
||||
}
|
||||
for key := range p.noWant {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
|
||||
delete(p.noWant, key) // no longer assigned there: its removed want says so
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
@@ -101,6 +103,238 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
}
|
||||
|
||||
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||
type memEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
value []byte
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (e memEntry) Value() []byte { return e.value }
|
||||
func (e memEntry) Revision() uint64 { return e.revision }
|
||||
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
v, ok := m.values[key]
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return memEntry{value: v, revision: uint64(len(v))}, nil
|
||||
}
|
||||
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if uint64(len(m.values[key])) != revision {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
m.values[key] = value
|
||||
return uint64(len(value)), nil
|
||||
}
|
||||
|
||||
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
func (s memStore) put(state, key string, value []byte) error {
|
||||
s[state].values[key] = value
|
||||
return nil
|
||||
}
|
||||
func (s memStore) keys(state string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range s[state].values {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
|
||||
type fakeBus struct {
|
||||
calls chan string
|
||||
events []string
|
||||
}
|
||||
|
||||
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
b.calls <- fmt.Sprintf("%s %v", key, body)
|
||||
return json.RawMessage(`{"ok":true,"output":""}`), nil
|
||||
}
|
||||
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
return b.Call(node+"/"+key, body)
|
||||
}
|
||||
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
|
||||
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
|
||||
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
|
||||
// gives the want back to the rules. Each step is said as the event put.
|
||||
func putOfTheBus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
|
||||
puts, asksKV, wantsKV := mem(), mem(), mem()
|
||||
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
p := newPasser(kvs, now)
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||
}
|
||||
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
|
||||
world := func(bucket, key string, v any) {
|
||||
var data []byte
|
||||
switch x := v.(type) {
|
||||
case []byte:
|
||||
data = x
|
||||
default:
|
||||
data, _ = json.Marshal(x)
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
bkey, _ := delivery.BuilderBuildKey("nats", fp)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
|
||||
world(delivery.BuilderBuildsBucket, bkey, built)
|
||||
hkey, _ := delivery.HaveKey("novox", "nats")
|
||||
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
|
||||
builds := mem()
|
||||
builds.values[bkey] = built
|
||||
modules := mem()
|
||||
modules.values["nats"] = mod
|
||||
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
|
||||
s.now = func() time.Time { return now }
|
||||
was := answerWithin
|
||||
answerWithin = 0
|
||||
defer func() { answerWithin = was }()
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("a prefix of a fingerprint the builder never built was taken")
|
||||
}
|
||||
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
|
||||
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
|
||||
}
|
||||
if got, err := s.version("nats", "newest"); err != nil || got != fp {
|
||||
t.Errorf("newest reads %q, %v", got, err)
|
||||
}
|
||||
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
|
||||
t.Fatalf("put answered %+v", a)
|
||||
}
|
||||
pkey, _ := delivery.PutKey("novox", "nats")
|
||||
step := func() delivery.Put {
|
||||
t.Helper()
|
||||
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
|
||||
p.pass(ctx)
|
||||
got, err := delivery.DecodePut(puts.values[pkey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
|
||||
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
select {
|
||||
case call := <-bus.calls:
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
world(delivery.ControllerGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil || a.Ask.Options[0].Level != asks.Approve {
|
||||
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
|
||||
}
|
||||
if _, there := wantsKV.values["want.novox.nats"]; there {
|
||||
t.Fatal("the bus's want was written before the operator's warrant")
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
|
||||
t.Errorf("the warrant was recorded as %q", call)
|
||||
}
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, running)
|
||||
if got := step(); got.Step != delivery.PutHolding {
|
||||
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutReleased {
|
||||
t.Fatalf("released, the put stands at %s", got.Step)
|
||||
}
|
||||
listed, _ := s.doPuts(map[string]any{"module": "nats"})
|
||||
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
|
||||
t.Errorf("puts lists %+v", l)
|
||||
}
|
||||
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
|
||||
t.Errorf("the steps said as events: %v", bus.events)
|
||||
}
|
||||
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
|
||||
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
s.now = func() time.Time { return now.Add(time.Minute) }
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := step()
|
||||
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes2)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
|
||||
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
|
||||
wantsKV.values["want.novox.nats"] = oldWant
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.pass(ctx)
|
||||
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
|
||||
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
|
||||
@@ -146,6 +380,31 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
|
||||
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
|
||||
for i, c := range []struct {
|
||||
before map[string]delivery.Want
|
||||
out assign.Outcome
|
||||
line string // the line said in this pass, "" for none
|
||||
}{
|
||||
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
|
||||
{nil, notDone, ""},
|
||||
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
if got := strings.Join(logged, "\n"); got != c.line {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
|
||||
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
|
||||
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
|
||||
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
|
||||
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
|
||||
// caller waits on it.
|
||||
|
||||
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
|
||||
const PutEvent = "put"
|
||||
|
||||
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
|
||||
// the seat held once, and an event said. process.Conn, or a test's.
|
||||
type caller interface {
|
||||
Call(key string, body any) (json.RawMessage, error)
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
|
||||
// takes the snapshot again, without harm.
|
||||
type putActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
snapping map[string]bool
|
||||
snapped map[string]string
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
}
|
||||
|
||||
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
|
||||
func identity(key string, p delivery.Put) string {
|
||||
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// putsByModule is every put the assigner holds, by module then node.
|
||||
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
|
||||
out := map[string]map[string]delivery.Put{}
|
||||
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
put, err := delivery.DecodePut(value)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read, and is left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]delivery.Put{}
|
||||
}
|
||||
out[module][node] = put
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// puts takes every put that has not ended one step further.
|
||||
func (p *passer) puts(ctx context.Context) {
|
||||
if p.acts == nil {
|
||||
return
|
||||
}
|
||||
all := p.putsByModule()
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
modules := make([]string, 0, len(all))
|
||||
for m := range all {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
var manifest catalogue.Manifest
|
||||
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
|
||||
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||
manifest = got.manifest
|
||||
}
|
||||
}
|
||||
for node, put := range all[module] {
|
||||
if delivery.PutEnded(put.Step) {
|
||||
continue
|
||||
}
|
||||
key, _ := delivery.PutKey(node, module)
|
||||
id := identity(key, put)
|
||||
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||
next := assign.StepPut(world)
|
||||
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
|
||||
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||
next.Put.Composed = w.Composed
|
||||
}
|
||||
}
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
|
||||
}
|
||||
}
|
||||
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
|
||||
!strings.Contains(next.Put.Words, failed) {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
|
||||
}
|
||||
p.actOn(ctx, node, module, id, world, next)
|
||||
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
|
||||
if !written {
|
||||
continue // released or put again meanwhile: nothing is done for the put that no longer stands
|
||||
}
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
|
||||
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
want := assign.PutWant(next.Put)
|
||||
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
if next.Warranted {
|
||||
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalWant(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
return w
|
||||
}
|
||||
|
||||
// putWorld is what one step of a put reads from the world.
|
||||
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
|
||||
for _, m := range assignedHere {
|
||||
if m == module {
|
||||
w.Assigned = true
|
||||
}
|
||||
}
|
||||
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
|
||||
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
|
||||
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
|
||||
w.Build, w.BuildWords = b.State, b.Error
|
||||
} else {
|
||||
w.Build = "unreadable: " + err.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
w.Answer = assign.OptionYes
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
w.Answer = assign.OptionNo
|
||||
default:
|
||||
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hkey, err := delivery.HaveKey(node, module); err == nil {
|
||||
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
w.Have = &h
|
||||
}
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||
// from their new owner and nothing else changes.
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, granted
|
||||
}
|
||||
|
||||
// actOn does the acts a step asks for.
|
||||
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
|
||||
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
|
||||
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||
// raises it anew under its own id.
|
||||
akey := next.Put.Ask
|
||||
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||
if err != nil {
|
||||
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||
} else {
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
if next.Snapshot {
|
||||
p.acts.snap(ctx, id, node, module)
|
||||
}
|
||||
}
|
||||
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||
func (p *passer) performs(key string, act asks.Act) error {
|
||||
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.ID == assign.OptionYes {
|
||||
return o.Performs(act)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the ask %s offers no yes", key)
|
||||
}
|
||||
|
||||
// samePut says two puts say the same: their step, words and what they found.
|
||||
func samePut(a, b delivery.Put) bool {
|
||||
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
|
||||
}
|
||||
|
||||
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
|
||||
// released it since; and says it as the event put. It says whether it wrote.
|
||||
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
|
||||
kv := p.kvs[delivery.AssignerPutsBucket]
|
||||
if kv == nil {
|
||||
return false
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
|
||||
return false
|
||||
}
|
||||
now, err := delivery.DecodePut(e.Value())
|
||||
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
|
||||
return false // asked again or released meanwhile: the next pass reads that
|
||||
}
|
||||
data, err := delivery.EncodePut(next)
|
||||
if err != nil {
|
||||
p.logf("the put %s is not written: %v", key, err)
|
||||
return false
|
||||
}
|
||||
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.put(delivery.AssignerPutsBucket, key, data)
|
||||
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||
p.acts.say(key, next)
|
||||
return true
|
||||
}
|
||||
|
||||
// state is what this process knows of a put's acts.
|
||||
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.snapped[id], a.snapping[id], a.snapNo[id]
|
||||
}
|
||||
|
||||
// snap has the bus's snapshot taken once for a put, in the background.
|
||||
func (a *putActs) snap(ctx context.Context, id, node, module string) {
|
||||
a.mu.Lock()
|
||||
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.snapping[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
where, err := a.snapshot(ctx, node, module)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.snapping[id] = false
|
||||
if err != nil {
|
||||
a.snapNo[id] = err.Error()
|
||||
return
|
||||
}
|
||||
a.snapped[id] = where
|
||||
}()
|
||||
}
|
||||
|
||||
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
|
||||
var SnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
|
||||
var snapshotLook = 10 * time.Second
|
||||
|
||||
// backedUp is one module as the backup holder's `backed-up` answers it.
|
||||
type backedUp struct {
|
||||
Module string `json:"module"`
|
||||
Data []struct {
|
||||
Item string `json:"item"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
|
||||
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
|
||||
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
|
||||
asked := time.Now()
|
||||
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
|
||||
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
deadline := asked.Add(SnapshotWithin)
|
||||
for {
|
||||
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
|
||||
var modules []backedUp
|
||||
if json.Unmarshal(raw, &modules) == nil {
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
|
||||
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(snapshotLook):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *putActs) warrant(id, ask string) {
|
||||
a.mu.Lock()
|
||||
if a.warranted[id] {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
|
||||
func (a *putActs) warrantFailed(id string) string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.warrantNo[id]
|
||||
}
|
||||
|
||||
// putSaid is the body of the event put.
|
||||
type putSaid struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
Hold bool `json:"hold"`
|
||||
Step string `json:"step"`
|
||||
Words string `json:"words"`
|
||||
Composed string `json:"composed,omitempty"`
|
||||
}
|
||||
|
||||
// say says a put's step as the event put, once per step and words.
|
||||
func (a *putActs) say(key string, p delivery.Put) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || a.bus == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.said[key] == p.Step+p.Words {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.said[key] = p.Step + p.Words
|
||||
a.mu.Unlock()
|
||||
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
|
||||
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
|
||||
}
|
||||
}
|
||||
|
||||
// shortHex is a digest's first twelve hex digits.
|
||||
func shortHex(digest string) string {
|
||||
h := strings.TrimPrefix(digest, "sha256:")
|
||||
if len(h) > 12 {
|
||||
return h[:12]
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
|
||||
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
|
||||
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
|
||||
var answerWithin = 20 * time.Second
|
||||
|
||||
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
|
||||
type store interface {
|
||||
get(state, key string) ([]byte, uint64, bool, error)
|
||||
put(state, key string, value []byte) error
|
||||
keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
|
||||
type putTools struct {
|
||||
st store
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
// heard is every put the watch handed, by key; changed is closed and replaced at each.
|
||||
heard map[string][]byte
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func newPutTools(st store) *putTools {
|
||||
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// hear takes a put's change in from the watch.
|
||||
func (s *putTools) hear(key string, value []byte) {
|
||||
s.mu.Lock()
|
||||
s.heard[key] = value
|
||||
close(s.changed)
|
||||
s.changed = make(chan struct{})
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
|
||||
var (
|
||||
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
|
||||
)
|
||||
|
||||
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
|
||||
// of one the builder built, which must name one build alone.
|
||||
func (s *putTools) version(module, version string) (string, error) {
|
||||
v := strings.ToLower(strings.TrimSpace(version))
|
||||
switch {
|
||||
case v == "newest":
|
||||
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||
}
|
||||
if m.FP2 == "" {
|
||||
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
|
||||
}
|
||||
return m.FP2, nil
|
||||
case fullFP.MatchString(v):
|
||||
return v, nil
|
||||
case prefix.MatchString(v):
|
||||
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
|
||||
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
|
||||
}
|
||||
var found []string
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
|
||||
found = append(found, fp)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
sort.Strings(found)
|
||||
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
|
||||
}
|
||||
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
|
||||
}
|
||||
|
||||
// putAnswer is what assigner_put and assigner_puts answer of one put.
|
||||
type putAnswer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
delivery.Put
|
||||
Follow string `json:"follow,omitempty"`
|
||||
}
|
||||
|
||||
// doPut is the tool assigner_put.
|
||||
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("put says why: it is kept with the put and said in any ask")
|
||||
}
|
||||
hold := false
|
||||
switch arg(args, "hold") {
|
||||
case "true":
|
||||
hold = true
|
||||
case "", "false":
|
||||
default:
|
||||
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
|
||||
}
|
||||
fp, err := s.version(module, arg(args, "version"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now().UTC()
|
||||
words := fmt.Sprintf("asked by %s: %s", by, why)
|
||||
if raw, _, found, err := s.st.get(putsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
} else if found {
|
||||
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
|
||||
if before.FP == fp && before.Hold == hold {
|
||||
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
|
||||
}
|
||||
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
|
||||
}
|
||||
}
|
||||
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
|
||||
}
|
||||
|
||||
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
|
||||
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
|
||||
deadline := time.After(answerWithin)
|
||||
for {
|
||||
s.mu.Lock()
|
||||
raw, changed := s.heard[key], s.changed
|
||||
s.mu.Unlock()
|
||||
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
|
||||
p = now
|
||||
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
|
||||
return p // it waits on a person or the node, or ended
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-changed:
|
||||
case <-deadline:
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// followPut says how a caller follows a put after the answer.
|
||||
func followPut(node, module string) string {
|
||||
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
|
||||
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
|
||||
}
|
||||
|
||||
// doRelease is the tool assigner_release.
|
||||
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept with the put")
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
|
||||
}
|
||||
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: p}, nil
|
||||
}
|
||||
|
||||
// doPuts is the tool assigner_puts.
|
||||
func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(putsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []putAnswer{}
|
||||
for _, key := range keys {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, putAnswer{Node: node, Module: module, Put: p})
|
||||
}
|
||||
return map[string]any{"puts": out}, nil
|
||||
}
|
||||
|
||||
func arg(args map[string]any, name string) string {
|
||||
switch v := args[name].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case bool:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The tools' names, as the manifest lists them.
|
||||
const (
|
||||
toolPut = "assigner_put"
|
||||
toolRelease = "assigner_release"
|
||||
toolPuts = "assigner_puts"
|
||||
)
|
||||
|
||||
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
|
||||
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
p := map[string]any{"type": "string", "description": description}
|
||||
if slices.Contains(switches, name) {
|
||||
p["enum"] = []string{"true", "false"}
|
||||
}
|
||||
props[name] = p
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
out["required"] = required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
|
||||
func (s *putTools) tools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
|
||||
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
|
||||
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
|
||||
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
|
||||
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine, where the module is assigned",
|
||||
"module": "the module",
|
||||
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
|
||||
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
|
||||
"why": "why, kept with the put and said in any ask",
|
||||
}, []string{"node", "module", "version", "why"}, "hold"),
|
||||
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
|
||||
[]string{"node", "module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||
"its time and words. Narrowed by machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||
Run: s.doPuts},
|
||||
}
|
||||
}
|
||||
|
||||
// runtimeStore is the runtime's states.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) put(state, key string, value []byte) error {
|
||||
_, err := stdio.State(state).Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
|
||||
func serveTools() error {
|
||||
s := newPutTools(runtimeStore{})
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
|
||||
s.hear(c.Key, c.Value)
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
return stdio.Serve("", s.tools())
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
func shortFP(fp string) string {
|
||||
part, err := delivery.FingerprintKeyPart(fp)
|
||||
if err != nil || len(part) < 12 {
|
||||
return fp
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -146,6 +146,44 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
for _, bus := range []bool{false, true} {
|
||||
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||
if err != nil {
|
||||
t.Fatalf("bus %v: %v", bus, err)
|
||||
}
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||
yes := a.Ask.Options[0]
|
||||
switch {
|
||||
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||
}
|
||||
later := put
|
||||
later.At = now.Add(time.Minute)
|
||||
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
t.Errorf("bus: the yes performs another composition's act")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
|
||||
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
|
||||
// the module from its asking until it ends (Module.Puts).
|
||||
|
||||
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
|
||||
// before it is refused.
|
||||
const ComposeWithin = 10 * time.Minute
|
||||
|
||||
// PutWorld is what one step of a put reads.
|
||||
type PutWorld struct {
|
||||
Node, Module string
|
||||
Put delivery.Put
|
||||
// Assigned says the module stands assigned to the node.
|
||||
Assigned bool
|
||||
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
|
||||
Bus, Recorded bool
|
||||
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *delivery.ComposedDigest
|
||||
Granted bool
|
||||
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
|
||||
// that ask expired unanswered.
|
||||
Answer string
|
||||
AskExpired bool
|
||||
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
|
||||
// asked; SnapshotFailed why it was not taken.
|
||||
Snapshot string
|
||||
Snapshotting bool
|
||||
SnapshotFailed string
|
||||
// Have is what the node says it runs of the module; nil when it says nothing.
|
||||
Have *delivery.Have
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
|
||||
type PutNext struct {
|
||||
Put delivery.Put
|
||||
// Ask writes the put's ask about the move (PutAsk).
|
||||
Ask bool
|
||||
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
|
||||
Snapshot bool
|
||||
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
|
||||
Warranted bool
|
||||
}
|
||||
|
||||
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
|
||||
func StepPut(w PutWorld) PutNext {
|
||||
p := w.Put
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return PutNext{Put: p}
|
||||
}
|
||||
say := func(step, format string, args ...any) PutNext {
|
||||
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
|
||||
case "":
|
||||
if p.FP != w.Newest {
|
||||
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||
}
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||
default:
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
|
||||
}
|
||||
since := w.Now
|
||||
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
|
||||
since = p.Steps[len(p.Steps)-1].At
|
||||
}
|
||||
if w.Now.Sub(since) > ComposeWithin {
|
||||
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
|
||||
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
|
||||
}
|
||||
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
|
||||
}
|
||||
d := *w.Composed
|
||||
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
|
||||
return PutNext{Put: applied(w, p)}
|
||||
}
|
||||
p.Composed = d.Digest
|
||||
if needsPerson(w, d) {
|
||||
switch w.Answer {
|
||||
case OptionNo:
|
||||
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
|
||||
case OptionYes:
|
||||
default:
|
||||
if w.AskExpired {
|
||||
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
|
||||
}
|
||||
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
|
||||
if p.Ask == "" {
|
||||
next.Ask = true
|
||||
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
|
||||
}
|
||||
return next
|
||||
}
|
||||
if w.Bus {
|
||||
switch {
|
||||
case w.Snapshot != "":
|
||||
p.Snapshot = w.Snapshot
|
||||
case w.SnapshotFailed != "":
|
||||
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
|
||||
case w.Snapshotting:
|
||||
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
default:
|
||||
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
next.Snapshot = true
|
||||
return next
|
||||
}
|
||||
}
|
||||
}
|
||||
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
|
||||
if p.Hold {
|
||||
words += ", held until released"
|
||||
}
|
||||
if p.Snapshot != "" {
|
||||
words += "; the streams were snapshotted: " + p.Snapshot
|
||||
}
|
||||
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
|
||||
next.Warranted = w.Bus && p.Ask != ""
|
||||
return next
|
||||
}
|
||||
|
||||
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
|
||||
func applied(w PutWorld, p delivery.Put) delivery.Put {
|
||||
h := w.Have
|
||||
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
|
||||
return p
|
||||
}
|
||||
switch {
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
|
||||
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
|
||||
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
|
||||
case failed(*h):
|
||||
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
|
||||
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
|
||||
// container, move data, or that cannot be judged (§14).
|
||||
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
|
||||
runs := w.Have != nil && w.Have.FP == w.Put.FP
|
||||
if runs && w.Have.Composed == d.Digest {
|
||||
return false // the node runs this composition already: nothing moves
|
||||
}
|
||||
if w.Bus {
|
||||
return !(runs && d.UserListOnly)
|
||||
}
|
||||
if !w.Recorded {
|
||||
return false
|
||||
}
|
||||
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
|
||||
func PutWant(p delivery.Put) delivery.Want {
|
||||
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
|
||||
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
|
||||
switch p.Step {
|
||||
case delivery.PutWanted, delivery.PutHolding:
|
||||
w := PutWant(p)
|
||||
return &w, true
|
||||
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
|
||||
return nil, false
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
|
||||
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
|
||||
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
|
||||
func PutAsking(p delivery.Put) int {
|
||||
if n := p.At.UnixMilli(); n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
|
||||
func PutAct(node, module, fp, composed string) asks.Act {
|
||||
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||
}
|
||||
|
||||
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
|
||||
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
|
||||
// channel that proves who answered takes it (ADR 0259), never the view.
|
||||
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
|
||||
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + d.Words,
|
||||
delivery.JudgedNeither: "recreates no container and moves no data",
|
||||
delivery.JudgedUnchanged: "changes nothing the node runs",
|
||||
}[d.Judgement]
|
||||
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
|
||||
expires := now.Add(7 * 24 * time.Hour)
|
||||
if bus {
|
||||
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
yes.Level, yes.Binds = asks.Approve, binds
|
||||
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
|
||||
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
|
||||
expires = now.Add(asks.ApproveLasts)
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, PutAsking(p)),
|
||||
Headline: fmt.Sprintf("Put %s on %s?", module, node),
|
||||
Explanation: explanation,
|
||||
Options: []asks.Option{yes,
|
||||
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
|
||||
Who: asks.Operator,
|
||||
Expires: expires,
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}, Asking: PutAsking(p)}, nil
|
||||
}
|
||||
@@ -44,6 +44,8 @@ type Module struct {
|
||||
Kept bool
|
||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
}
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
@@ -224,6 +226,29 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next delivery.Want
|
||||
p, put := m.Puts[node]
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
||||
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
||||
// without hold is not kept this way: its put takes it further once heard.
|
||||
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
||||
Step: delivery.PutHolding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
if want != nil {
|
||||
w := *want
|
||||
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
||||
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
|
||||
w.Composed = ul.Composed
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
@@ -266,7 +291,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && before.FP == target && before.Composed == d.Digest {
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
@@ -308,8 +333,18 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
|
||||
h, there := n.Haves[node][m.Name]
|
||||
switch {
|
||||
case there && h.Phase != delivery.PhaseRemoved && !held:
|
||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||
case had && (!there || h.Phase == delivery.PhaseRemoved):
|
||||
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
||||
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
||||
// would have its node agent remove the module and stop holding its composition's resources back.
|
||||
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
|
||||
if !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -345,6 +380,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
}
|
||||
|
||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||
// moving). A failed health while it
|
||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||
@@ -475,17 +515,8 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node: its own `<node>.<module>` or a process's
|
||||
// `<node>.<module>.<process>`. A grant that names no account (a module that does not speak on the bus) names every node.
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
if len(g.Accounts) == 0 {
|
||||
return true
|
||||
}
|
||||
own := node + "." + module
|
||||
for _, a := range g.Accounts {
|
||||
if a == own || strings.HasPrefix(a, own+".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -384,6 +384,142 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
||||
}
|
||||
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
|
||||
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
|
||||
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
step string
|
||||
wanted map[string]delivery.Want
|
||||
want *delivery.Want // nil: nothing written on ace
|
||||
}{
|
||||
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
|
||||
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||
} {
|
||||
p := put
|
||||
p.Step = c.step
|
||||
m.Puts = map[string]delivery.Put{"ace": p}
|
||||
if c.step == "" {
|
||||
m.Puts = nil
|
||||
}
|
||||
out := Wants(m, n, c.wanted, nil)
|
||||
got, wrote := out.Wants["ace"]
|
||||
switch {
|
||||
case c.want == nil && wrote:
|
||||
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
|
||||
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
|
||||
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
|
||||
}
|
||||
}
|
||||
userList := "sha256:" + strings.Repeat("7", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
held := put
|
||||
held.Step = delivery.PutHolding
|
||||
m.Puts = map[string]delivery.Put{"ace": held}
|
||||
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
|
||||
// operator's yes, the bus's snapshot, the want, and the have.
|
||||
func stepsOfAPut(t *testing.T) {
|
||||
t.Helper()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
|
||||
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
bus := func() PutWorld {
|
||||
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
|
||||
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
world func() PutWorld
|
||||
step string
|
||||
says string
|
||||
ask, snap bool
|
||||
warranted bool
|
||||
}{
|
||||
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
|
||||
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
|
||||
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
|
||||
{"none within the bound: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Composed = nil
|
||||
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
|
||||
return w
|
||||
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
|
||||
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
|
||||
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
|
||||
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
|
||||
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
|
||||
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
|
||||
{"snapshotted: the want, warranted", func() PutWorld {
|
||||
w := bus()
|
||||
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
|
||||
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
|
||||
return w
|
||||
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
|
||||
{"a user-list revision of the bus needs nobody", func() PutWorld {
|
||||
w := bus()
|
||||
ul := d
|
||||
ul.UserListOnly = true
|
||||
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutWanted, "held until released", false, false, false},
|
||||
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
|
||||
{"a recorded module judged to recreate asks", func() PutWorld {
|
||||
w := bus()
|
||||
w.Bus, w.Recorded = false, true
|
||||
r := d
|
||||
r.Judgement = delivery.JudgedRecreates
|
||||
w.Composed = &r
|
||||
return w
|
||||
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||
{"wanted, the node runs it: holding", func() PutWorld {
|
||||
w := bus()
|
||||
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||
w.Put.Composed = d.Digest
|
||||
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutHolding, "held there until released", false, false, false},
|
||||
{"wanted without hold: done", func() PutWorld {
|
||||
w := bus()
|
||||
w.Put.Hold = false
|
||||
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||
w.Put.Composed = d.Digest
|
||||
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutDone, "the rules write its want again", false, false, false},
|
||||
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
next := StepPut(c.world())
|
||||
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||
}
|
||||
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
|
||||
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
|
||||
c.ask, c.snap, c.warranted)
|
||||
}
|
||||
if next.Ask && next.Put.Ask == "" {
|
||||
t.Errorf("put, %s: an ask without its key", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
||||
@@ -731,6 +867,33 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
||||
}
|
||||
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
|
||||
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
|
||||
// runs the module is on neither list, its removed want left as it is.
|
||||
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
before map[string]delivery.Want
|
||||
want map[string]delivery.Want
|
||||
}{
|
||||
{"no key: nothing written", nil, map[string]delivery.Want{}},
|
||||
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
|
||||
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
|
||||
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
|
||||
map[string]delivery.Want{}},
|
||||
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
|
||||
} {
|
||||
out := Wants(m, n, c.before, nil)
|
||||
if len(out.Wants) != len(c.want) {
|
||||
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
|
||||
continue
|
||||
}
|
||||
for node, w := range c.want {
|
||||
if !delivery.SameWant(normal(out.Wants[node]), w) {
|
||||
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
m.FP = ""
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
||||
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
||||
|
||||
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
|
||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
func TestManifest(t *testing.T) {
|
||||
m := load(t)
|
||||
if m.Module != "mesh-delivery-assigner" {
|
||||
@@ -42,7 +42,7 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -56,7 +56,16 @@ func TestManifest(t *testing.T) {
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
}
|
||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
|
||||
// calls the controller's warranted and the bus node's backup holder.
|
||||
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
|
||||
}
|
||||
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
t.Errorf("emits = %v, want put", m.Emits)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,24 @@
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.warranted",
|
||||
"seat:node-backup.now",
|
||||
"seat:node-backup.backed-up"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
],
|
||||
"tools": [
|
||||
"assigner_put",
|
||||
"assigner_release",
|
||||
"assigner_puts"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 }
|
||||
],
|
||||
"own-secrets": {
|
||||
@@ -71,7 +85,10 @@
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-assigner",
|
||||
"binary": "mesh-delivery-assigner"
|
||||
"binary": "mesh-delivery-assigner",
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
|
||||
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
|
||||
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
|
||||
// other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueAsker writes a queues value and waits for the builder's answer to it.
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
{Name: build.AskAgainTool,
|
||||
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
|
||||
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
|
||||
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
|
||||
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := askAgain(key, value); err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
|
||||
}
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
|
||||
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
|
||||
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
|
||||
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
|
||||
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
language := build.LanguageOf(str(args, "language"))
|
||||
if language == "" {
|
||||
language = build.LanguageOf(str(args, "seat"))
|
||||
}
|
||||
if language == "" {
|
||||
language = delivery.AskGo
|
||||
}
|
||||
limit, _ := args["limit"].(float64)
|
||||
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return queue(key, value)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
answered := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
|
||||
select {
|
||||
case answered <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
|
||||
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
|
||||
return e.Value, nil
|
||||
}
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-answered:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) build.StateRead {
|
||||
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return e.Value, time.Time{}, nil
|
||||
}, Written: func(key string) time.Time {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return written(s, key, e.Revision)
|
||||
}}
|
||||
}
|
||||
|
||||
// written is when the key's revision was written, from its history, or zero.
|
||||
func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
||||
h, err := s.History(key)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, r := range h {
|
||||
if r.Revision == revision {
|
||||
return r.At
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func main() {
|
||||
steps := map[string]build.StateRead{}
|
||||
for _, bm := range build.StuckBuildModules {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,11 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return settled(ctx, w, each)
|
||||
}
|
||||
|
||||
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
|
||||
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
@@ -4,73 +4,303 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||
// (issue 1910): read once, at the take-over of a key at start, never polled.
|
||||
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
|
||||
// be answered (issue 1945). Never polled.
|
||||
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||
|
||||
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
seen, err := h.seen(ctx, a, false)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
return seen.Step || seen.OnStream, nil
|
||||
}
|
||||
|
||||
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
|
||||
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
|
||||
seen, err := h.seen(ctx, a, true)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return "", err
|
||||
}
|
||||
return seen.Why(), nil
|
||||
}
|
||||
|
||||
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
|
||||
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
|
||||
const (
|
||||
maxOnSubject = 5000
|
||||
maxSteps = 200
|
||||
)
|
||||
|
||||
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
|
||||
// the ask is on its stream.
|
||||
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
|
||||
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return seen, err
|
||||
}
|
||||
for range keys.Keys() {
|
||||
keys.Stop()
|
||||
return true, nil // a node took it
|
||||
seen.Step = true // a node took it
|
||||
return seen, nil
|
||||
}
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
seen.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return seen, nil
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return seen, err
|
||||
}
|
||||
var ahead []build.Queued
|
||||
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seen.OnStream, seen.PutAt = true, q.PutAt
|
||||
return false
|
||||
}
|
||||
ahead = append(ahead, q)
|
||||
return true
|
||||
})
|
||||
if err != nil || !seen.OnStream || !later {
|
||||
return seen, err
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
}
|
||||
seen.Ahead = ahead
|
||||
since := seen.At
|
||||
if since.IsZero() {
|
||||
since = seen.PutAt
|
||||
}
|
||||
for _, q := range ahead {
|
||||
if q.At.Before(since) {
|
||||
since = q.At
|
||||
}
|
||||
}
|
||||
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
// stream is the build seat's stream of language, its name and the subject of kind on it.
|
||||
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
|
||||
subject, err := delivery.AskSubject(language, kind)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
}
|
||||
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
||||
// messages by sequence and nothing wider (issue 1910).
|
||||
name, err := delivery.AskStream(a.Language)
|
||||
name, err := delivery.AskStream(language)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return nil, "", "", err
|
||||
}
|
||||
stream, err := h.js.Stream(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
|
||||
// instead, each a direct read that consumes nothing.
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
|
||||
m, err := stream.GetMsg(ctx, seq)
|
||||
return stream, name, subject, err
|
||||
}
|
||||
|
||||
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
|
||||
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
|
||||
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
|
||||
// none consumed.
|
||||
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
|
||||
for seq, n := uint64(1), 0; ; n++ {
|
||||
if n >= maxOnSubject {
|
||||
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
|
||||
}
|
||||
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
continue // taken and acknowledged, or removed
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
|
||||
return true, nil
|
||||
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
|
||||
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
|
||||
q.Kind = kind
|
||||
}
|
||||
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
|
||||
q.At = q.PutAt
|
||||
}
|
||||
if !each(q) {
|
||||
return nil
|
||||
}
|
||||
seq = m.Sequence + 1
|
||||
}
|
||||
}
|
||||
|
||||
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
|
||||
// ask id and is left out).
|
||||
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
|
||||
all, err := steps.ListKeys(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return map[string][]string{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer all.Stop()
|
||||
out := map[string][]string{}
|
||||
for k := range all.Keys() {
|
||||
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
|
||||
out[id] = append(out[id], k)
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stepState is the state of the first of keys a node wrote, "" for none.
|
||||
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
|
||||
for _, k := range keys {
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if s, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
return s.State
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
|
||||
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
|
||||
// (mesh-sdk buildstep).
|
||||
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
|
||||
var ids []string
|
||||
for id := range taken {
|
||||
if build.AskedAt(id).After(since) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
|
||||
read := 0
|
||||
for _, id := range ids {
|
||||
for _, k := range taken[id] {
|
||||
if read >= maxSteps {
|
||||
return nil
|
||||
}
|
||||
read++
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ok || !build.Proves(stepKind, kind) {
|
||||
continue
|
||||
}
|
||||
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
|
||||
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
|
||||
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var seq uint64
|
||||
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seq = q.Seq
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); err != nil || seq == 0 {
|
||||
return err
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
|
||||
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
|
||||
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
|
||||
q := build.SeatQueue{Language: language, Note: build.QueueNote}
|
||||
module, err := delivery.BuildModuleOf(language)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
|
||||
q.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
q.Note = name + " does not exist"
|
||||
return q, nil
|
||||
}
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
for _, kind := range delivery.AskKinds {
|
||||
subject, _ := delivery.AskSubject(language, kind)
|
||||
var ahead []build.Queued
|
||||
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
|
||||
ahead = append(ahead, m)
|
||||
return len(ahead) < limit
|
||||
}); err != nil {
|
||||
return q, err
|
||||
}
|
||||
if len(ahead) == 0 {
|
||||
continue
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
}
|
||||
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
|
||||
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
@@ -24,6 +24,8 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -162,11 +164,29 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
|
||||
const trunkHead = "3333333333333333333333333333333333333333"
|
||||
|
||||
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
|
||||
func (b *bed) trunk(t *testing.T, head string) uint64 {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
modules jetstream.KeyValue
|
||||
candidates jetstream.KeyValue
|
||||
source jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
tools map[string]jetstream.KeyValue
|
||||
instances jetstream.KeyValue
|
||||
@@ -243,6 +263,7 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
}
|
||||
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
@@ -513,7 +534,9 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
||||
|
||||
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
||||
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
|
||||
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
|
||||
// fingerprint first appeared at).
|
||||
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
||||
b := startBed(t)
|
||||
other := "src2:" + strings.Repeat("e", 64)
|
||||
@@ -530,7 +553,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
|
||||
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -550,12 +573,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
defer b.fake.mu.Unlock()
|
||||
found := false
|
||||
for _, f := range b.fake.fetches {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -603,7 +626,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
|
||||
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -801,6 +824,34 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
||||
}
|
||||
}
|
||||
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
|
||||
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
|
||||
held := heldOnTheBus{b.js}
|
||||
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
|
||||
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queued := func() []build.QueuedState {
|
||||
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
|
||||
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
|
||||
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
|
||||
}
|
||||
return q.Kinds[0].Messages
|
||||
}
|
||||
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
|
||||
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
|
||||
}
|
||||
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
|
||||
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
|
||||
}
|
||||
if err := held.Drop(b.ctx, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
|
||||
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
|
||||
}
|
||||
b.stop()
|
||||
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
||||
b.fake.open()
|
||||
|
||||
@@ -13,15 +13,19 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -121,29 +125,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
builds, err := bucket(delivery.BuilderBuildsBucket)
|
||||
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
|
||||
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
|
||||
bound := defaultStepBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||
if bound, err = time.ParseDuration(v); err != nil {
|
||||
@@ -166,23 +161,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
CheckAfter: bound,
|
||||
Toolchains: tools,
|
||||
Manifests: manifestReader{manifests},
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||
errs := make(chan error, 6)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
b.Said = func(e build.AskedAgain) {
|
||||
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||
// is asked since R4g, ADR 0524 §7).
|
||||
switch kv, err := js.KeyValue(ctx, instancesBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
|
||||
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
|
||||
case kv == nil:
|
||||
default:
|
||||
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||
b.Answered = says.Answered
|
||||
@@ -262,14 +270,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
b.ForgetCandidate(e.Key())
|
||||
}
|
||||
}
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
return err
|
||||
if candidates != nil {
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
}
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
}
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
}
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
@@ -282,3 +312,50 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||
// delivery.SourceHistory of them: what a candidate's base revision names.
|
||||
type sourceHeads struct {
|
||||
mu sync.Mutex
|
||||
byRev map[uint64]map[string]string
|
||||
}
|
||||
|
||||
// put takes one revision in and answers whether it is new.
|
||||
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
|
||||
if !opOf(e) {
|
||||
return false
|
||||
}
|
||||
src, err := delivery.DecodeSource(e.Value())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.byRev == nil {
|
||||
s.byRev = map[uint64]map[string]string{}
|
||||
}
|
||||
if _, had := s.byRev[e.Revision()]; had {
|
||||
return false
|
||||
}
|
||||
s.byRev[e.Revision()] = src.Heads
|
||||
for len(s.byRev) > delivery.SourceHistory {
|
||||
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
|
||||
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
heads, ok := s.byRev[rev]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
|
||||
}
|
||||
for r, head := range heads {
|
||||
if delivery.SameRepository(r, repository) {
|
||||
return head, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
|
||||
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
|
||||
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
|
||||
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
|
||||
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
|
||||
// and opened again at the next event that needs it where there is one.
|
||||
|
||||
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
|
||||
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
|
||||
// and a candidate's preview waits, saying why.
|
||||
type trunkSource struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
heads *sourceHeads
|
||||
moved func(context.Context)
|
||||
errs chan<- error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
}
|
||||
|
||||
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
|
||||
func (t *trunkSource) ensure() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.opened {
|
||||
return nil
|
||||
}
|
||||
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
}
|
||||
t.opened = true
|
||||
go func() {
|
||||
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if t.heads.put(e) && t.moved != nil {
|
||||
t.moved(t.ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// start opens the source at start; one that cannot be is said, and the builder goes on.
|
||||
func (t *trunkSource) start() {
|
||||
if err := t.ensure(); err != nil {
|
||||
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
|
||||
}
|
||||
}
|
||||
|
||||
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
|
||||
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if err := t.ensure(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t.heads.head(ctx, rev, repository)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
|
||||
type queueReader interface {
|
||||
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
|
||||
}
|
||||
|
||||
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
|
||||
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
|
||||
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
|
||||
// instance's) or deleted is left.
|
||||
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
a, err := build.DecodeQueueAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
a.Error = err.Error()
|
||||
}
|
||||
a.Queue, a.Answered = &q, time.Now().UTC()
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
}
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
r, err := build.DecodeReAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
} else {
|
||||
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -28,8 +28,11 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -38,14 +41,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -67,20 +70,31 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
open := !deleted && openAssignment(value)
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.standing[key] == open {
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = true
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -111,37 +125,58 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return false
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -155,17 +190,19 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,10 +3,19 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -44,10 +53,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -63,20 +72,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -88,25 +97,132 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,11 +4,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -62,11 +64,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
|
||||
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
|
||||
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
|
||||
const AskedAgainEvent = "build.asked-again"
|
||||
|
||||
// AskedAgain is the body of AskedAgainEvent.
|
||||
type AskedAgain struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
BuildModule string `json:"build_module"`
|
||||
Language string `json:"language"`
|
||||
Old string `json:"old"`
|
||||
New string `json:"new"`
|
||||
Asking int `json:"asking"`
|
||||
Askings int `json:"askings"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
|
||||
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
|
||||
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
|
||||
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
|
||||
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
|
||||
// five hours, each counted as waiting, so the ask was never asked again).
|
||||
type AskSeen struct {
|
||||
Stream string
|
||||
// Kind is the ask's kind, At when it was asked (its id's time).
|
||||
Kind string
|
||||
At time.Time
|
||||
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
|
||||
Step bool
|
||||
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
|
||||
OnStream bool
|
||||
PutAt time.Time
|
||||
// Ahead are the messages of its subject before it on the stream.
|
||||
Ahead []Queued
|
||||
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
|
||||
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
|
||||
Last *Taken
|
||||
}
|
||||
|
||||
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
|
||||
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
|
||||
type Queued struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
PutAt time.Time `json:"put"`
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
|
||||
type Taken struct {
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
Node string `json:"node"`
|
||||
Started time.Time `json:"started"`
|
||||
}
|
||||
|
||||
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
|
||||
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
|
||||
// its worker had none of waiting's kind left to give.
|
||||
func Proves(taken, waiting string) bool {
|
||||
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
|
||||
return t >= 0 && w >= 0 && t >= w
|
||||
}
|
||||
|
||||
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
|
||||
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
|
||||
// its age, and is not counted.
|
||||
func (s AskSeen) Waits(q Queued) bool {
|
||||
return q.Step == "" && !s.passed(q.At)
|
||||
}
|
||||
|
||||
func (s AskSeen) passed(at time.Time) bool {
|
||||
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
|
||||
}
|
||||
|
||||
// Waiting is how many messages ahead still wait to be delivered (Waits).
|
||||
func (s AskSeen) Waiting() int {
|
||||
n := 0
|
||||
for _, q := range s.Ahead {
|
||||
if s.Waits(q) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
|
||||
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
|
||||
// ADR 0525 §9).
|
||||
func (s AskSeen) Why() string {
|
||||
switch {
|
||||
case s.Step:
|
||||
return ""
|
||||
case !s.OnStream:
|
||||
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
|
||||
case s.Waiting() > 0 || !s.passed(s.at()):
|
||||
return ""
|
||||
default:
|
||||
behind := ""
|
||||
if n := len(s.Ahead); n > 0 {
|
||||
behind = fmt.Sprintf(" behind %d given up on or taken", n)
|
||||
}
|
||||
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
|
||||
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
|
||||
func (s AskSeen) at() time.Time {
|
||||
if s.At.IsZero() {
|
||||
return s.PutAt
|
||||
}
|
||||
return s.At
|
||||
}
|
||||
|
||||
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
|
||||
func AskedAt(id string) time.Time {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.UnixMilli(ms)
|
||||
}
|
||||
|
||||
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
|
||||
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
|
||||
type Dropper interface {
|
||||
Drop(ctx context.Context, a delivery.StepAsk) error
|
||||
}
|
||||
|
||||
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
|
||||
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
d, ok := b.Gone.(Dropper)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
prior := ask
|
||||
prior.ID = old
|
||||
if err := d.Drop(ctx, prior); err != nil {
|
||||
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
}
|
||||
return DefaultAskings
|
||||
}
|
||||
|
||||
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
|
||||
|
||||
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
|
||||
func askingOf(step, buildModule string) int {
|
||||
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
|
||||
if m[1] == buildModule {
|
||||
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
|
||||
// languages whose lane waits, none when no lane of this builder waits on the key.
|
||||
func (b *Builder) AskAgain(key, why string) []string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
var asked []string
|
||||
for language, c := range b.waiting[key] {
|
||||
select {
|
||||
case c <- why:
|
||||
default: // one is already asked
|
||||
}
|
||||
asked = append(asked, language)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
|
||||
func (b *Builder) Waiting() map[string][]string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
out := map[string][]string{}
|
||||
for key, ls := range b.waiting {
|
||||
for l := range ls {
|
||||
out[key] = append(out[key], l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Builder) listen(key, language string) chan string {
|
||||
b.init()
|
||||
c := make(chan string, 1)
|
||||
b.mu.Lock()
|
||||
if b.waiting[key] == nil {
|
||||
b.waiting[key] = map[string]chan string{}
|
||||
}
|
||||
b.waiting[key][language] = c
|
||||
b.mu.Unlock()
|
||||
return c
|
||||
}
|
||||
|
||||
func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
b.mu.Lock()
|
||||
if b.waiting[key][language] == c {
|
||||
delete(b.waiting[key], language)
|
||||
if len(b.waiting[key]) == 0 {
|
||||
delete(b.waiting, key)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
|
||||
func (h byHand) Error() string { return "asked again by hand: " + h.why }
|
||||
|
||||
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
|
||||
// asked it again (AskAgain); it answers the step and the ask that has it.
|
||||
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
|
||||
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
}
|
||||
asking++
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
|
||||
if b.Said != nil {
|
||||
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
|
||||
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
|
||||
}
|
||||
default:
|
||||
return step, ask, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
|
||||
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
|
||||
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
|
||||
hand, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
go func() {
|
||||
select {
|
||||
case why := <-again:
|
||||
cancel(byHand{why})
|
||||
case <-hand.Done():
|
||||
}
|
||||
}()
|
||||
wait := hand
|
||||
if b.Gone != nil && b.CheckAfter > 0 {
|
||||
var stop context.CancelFunc
|
||||
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
var h byHand
|
||||
if errors.As(context.Cause(hand), &h) {
|
||||
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
|
||||
}
|
||||
if errors.Is(context.Cause(wait), errCheck) {
|
||||
why, gerr := b.Gone.Gone(ctx, ask)
|
||||
if gerr != nil {
|
||||
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
|
||||
return step, "", errCheck
|
||||
}
|
||||
if why == "" {
|
||||
return step, "", errCheck
|
||||
}
|
||||
return step, why, errAgain
|
||||
}
|
||||
return step, "", err
|
||||
}
|
||||
@@ -73,9 +73,17 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||
}
|
||||
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
|
||||
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
|
||||
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
|
||||
base, err := b.trunkHead(ctx, c.Base, p.Repository)
|
||||
if err != nil {
|
||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
|
||||
continue
|
||||
}
|
||||
b.mu.Lock()
|
||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, name, m)
|
||||
@@ -106,3 +114,18 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
|
||||
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if b.Trunk == nil {
|
||||
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
|
||||
}
|
||||
head, err := b.Trunk(ctx, rev, repository)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
|
||||
}
|
||||
if head == "" {
|
||||
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
@@ -2,9 +2,13 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -40,11 +44,74 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
|
||||
}
|
||||
|
||||
const (
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
|
||||
// commit its fingerprint first appeared at.
|
||||
trunkHead = "3333333333333333333333333333333333333333"
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
|
||||
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
|
||||
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w struct {
|
||||
Source struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
} `json:"source"`
|
||||
Candidate json.RawMessage `json:"candidate"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w.Source.Rev, w.Source.Heads, c, w.Candidate
|
||||
}
|
||||
|
||||
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
||||
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
|
||||
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
|
||||
{
|
||||
rev, heads, c, raw := indexerCandidate(t)
|
||||
p := c.Pulls[0]
|
||||
cm := p.Fps["avahi"]
|
||||
ir := newRig(t, goBundle)
|
||||
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
|
||||
if at != rev {
|
||||
return "", fmt.Errorf("source revision %d is not heard", at)
|
||||
}
|
||||
return heads[repository], nil
|
||||
}
|
||||
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
||||
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
|
||||
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ir.b.Module(context.Background(), "avahi", value)
|
||||
ir.b.Wait()
|
||||
ir.bm.mu.Lock()
|
||||
ir.bm.calls = nil
|
||||
ir.bm.mu.Unlock()
|
||||
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
|
||||
ir.b.Wait()
|
||||
fetches := ir.bm.callsOf("build_go_fetch")
|
||||
if len(fetches) != 1 {
|
||||
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
|
||||
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
|
||||
}
|
||||
}
|
||||
r := newRig(t, goBundle)
|
||||
trunkModule(t, r)
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||
@@ -58,8 +125,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
|
||||
t.Fatalf("%d fetches, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
|
||||
}
|
||||
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
||||
t.Errorf("build args = %+v", b)
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
|
||||
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
|
||||
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
|
||||
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
|
||||
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
|
||||
// there, as builder_ask_again does through reasks.
|
||||
const (
|
||||
SeatQueueTool = "builder_seat_queue"
|
||||
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
|
||||
QueuesState = "queues"
|
||||
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
|
||||
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
|
||||
DefaultQueueLimit = 50
|
||||
MaxQueueLimit = 500
|
||||
)
|
||||
|
||||
// QueueNote says what the read cannot show.
|
||||
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
|
||||
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
|
||||
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
|
||||
|
||||
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
|
||||
type QueueAsk struct {
|
||||
V int `json:"v"`
|
||||
Limit int `json:"limit"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Queue *SeatQueue `json:"queue,omitempty"`
|
||||
}
|
||||
|
||||
// SeatQueue is what the builder read of one build seat's stream.
|
||||
type SeatQueue struct {
|
||||
Language string `json:"language"`
|
||||
Stream string `json:"stream"`
|
||||
Messages uint64 `json:"messages"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
Kinds []KindQueue `json:"kinds"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
|
||||
// last a build node took of the kind (or of one pulled after it), and each message.
|
||||
type KindQueue struct {
|
||||
Kind string `json:"kind"`
|
||||
Read int `json:"read"`
|
||||
Waiting int `json:"waiting"`
|
||||
GivenUp int `json:"given_up_or_taken"`
|
||||
Last *Taken `json:"last_taken,omitempty"`
|
||||
Messages []QueuedState `json:"messages"`
|
||||
}
|
||||
|
||||
// QueuedState is one message as the tool shows it.
|
||||
type QueuedState struct {
|
||||
Queued
|
||||
AgeSeconds int64 `json:"age_s"`
|
||||
Waits bool `json:"waits"`
|
||||
}
|
||||
|
||||
// NewKindQueue is seen's messages ahead as one kind's queue at now.
|
||||
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
|
||||
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
|
||||
for _, q := range seen.Ahead {
|
||||
w := seen.Waits(q)
|
||||
if w {
|
||||
k.Waiting++
|
||||
} else {
|
||||
k.GivenUp++
|
||||
}
|
||||
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
|
||||
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
|
||||
if _, err := delivery.AskStream(language); err != nil {
|
||||
return "", nil, fmt.Errorf("language: %w", err)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = DefaultQueueLimit
|
||||
}
|
||||
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
|
||||
return language, data, err
|
||||
}
|
||||
|
||||
// DecodeQueueAsk reads a queues value.
|
||||
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
|
||||
var a QueueAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
|
||||
func LanguageOf(arg string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
|
||||
}
|
||||
@@ -379,8 +379,9 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
|
||||
t.Fatal("no fetch")
|
||||
}
|
||||
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
|
||||
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
||||
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
|
||||
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -53,6 +53,21 @@ type Builder struct {
|
||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||
Gone interface {
|
||||
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
|
||||
}
|
||||
CheckAfter time.Duration
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -72,6 +87,8 @@ type Builder struct {
|
||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -88,7 +105,7 @@ func (b *Builder) init() {
|
||||
b.sem = make(chan struct{}, limit)
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held = map[string]bool{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -269,6 +286,13 @@ func (b *Builder) Reconsider(ctx context.Context) {
|
||||
for _, k := range sortedKeys(known) {
|
||||
b.consider(ctx, k, known[k])
|
||||
}
|
||||
b.TrunkMoved(ctx)
|
||||
}
|
||||
|
||||
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
|
||||
// revision it was previewed onto had no trunk head to be merged onto.
|
||||
func (b *Builder) TrunkMoved(ctx context.Context) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
cands := make([]delivery.Candidate, 0, len(b.cands))
|
||||
for _, c := range b.cands {
|
||||
@@ -663,12 +687,14 @@ type askNotes struct {
|
||||
b *Builder
|
||||
cl *Claim
|
||||
resumed map[string]string
|
||||
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
|
||||
step string
|
||||
}
|
||||
|
||||
func (b *Builder) notes(cl *Claim) *askNotes {
|
||||
n := &askNotes{b: b, cl: cl}
|
||||
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
||||
n.resumed = maps.Clone(cl.Build.Asks)
|
||||
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -766,7 +792,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
return
|
||||
}
|
||||
}
|
||||
step, err := b.Asks.Wait(ctx, ask)
|
||||
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
|
||||
if err != nil {
|
||||
l.err = fmt.Errorf("%s: %w", l.module, err)
|
||||
return
|
||||
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -176,6 +178,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
||||
r.b = &Builder{
|
||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
|
||||
}
|
||||
var err error
|
||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||
@@ -615,27 +618,286 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||
// again; the key is done when a node takes it.
|
||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
||||
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
||||
// drop from their stream (Dropper).
|
||||
type goneAsks struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
why func(a delivery.StepAsk) string
|
||||
dropped []string
|
||||
}
|
||||
|
||||
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.dropped = append(g.dropped, a.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
|
||||
g.mu.Lock()
|
||||
g.n++
|
||||
g.mu.Unlock()
|
||||
return g.why(a), nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) asked() int {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.n
|
||||
}
|
||||
|
||||
// waitForAsks waits until n asks were put.
|
||||
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask was put")
|
||||
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
return r.asks.Asks()
|
||||
}
|
||||
|
||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
|
||||
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
|
||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Run("waits in its stream: never asked again", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
waitForAsks(t, r, 1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
}
|
||||
if gone.asked() == 0 {
|
||||
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
}{
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) { // 1945
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Askings = c.askings
|
||||
var said []AskedAgain
|
||||
var mu sync.Mutex
|
||||
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
||||
first := ""
|
||||
if c.step != "" {
|
||||
first = "mvl0frv18d1960"
|
||||
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
if !c.byHand {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no lane of the key waits")
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.done {
|
||||
asks := waitForAsks(t, r, c.puts)
|
||||
if first == "" && !c.byHand {
|
||||
first = asks[0].ID
|
||||
}
|
||||
last := asks[len(asks)-1]
|
||||
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
|
||||
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
|
||||
}
|
||||
r.asks.End(r.answer(last))
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); n != c.puts {
|
||||
t.Errorf("%d asks put, want %d", n, c.puts)
|
||||
}
|
||||
if c.done && v.State != delivery.BuildDone {
|
||||
t.Errorf("the key = %+v, want done from the ask asked again", v)
|
||||
}
|
||||
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
|
||||
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
taken := func(kind string, at time.Time) *Taken {
|
||||
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
|
||||
}
|
||||
later := taken(delivery.AskDelivery, put.Add(time.Minute))
|
||||
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
|
||||
ahead := func(n int, step string) []Queued {
|
||||
var qs []Queued
|
||||
for i := range n {
|
||||
at := put.Add(-time.Duration(n-i) * time.Minute)
|
||||
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
|
||||
}
|
||||
return qs
|
||||
}
|
||||
seen := func(qs []Queued, last *Taken) AskSeen {
|
||||
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
seen AskSeen
|
||||
want string // a part of the why, "" when it can be answered
|
||||
waiting int
|
||||
}{
|
||||
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
|
||||
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
|
||||
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
|
||||
{"first in line, nothing taken since", seen(nil, nil), "", 0},
|
||||
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
|
||||
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
|
||||
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
|
||||
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
|
||||
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
|
||||
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
|
||||
} {
|
||||
got := c.seen.Why()
|
||||
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
|
||||
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
|
||||
}
|
||||
}
|
||||
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
|
||||
t.Errorf("AskedAt does not read an ask id's time")
|
||||
}
|
||||
})
|
||||
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
|
||||
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
state := func(vals map[string]string) StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) {
|
||||
var ks []string
|
||||
for k := range vals {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
return ks, nil
|
||||
}, Get: func(k string) (json.RawMessage, time.Time, error) {
|
||||
if v, ok := vals[k]; ok {
|
||||
return json.RawMessage(v), at, nil
|
||||
}
|
||||
return nil, time.Time{}, nil
|
||||
}}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||
done, _ := delivery.BuilderBuildKey("d", fp1)
|
||||
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
|
||||
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
}
|
||||
a := stuck[0].Asks
|
||||
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
|
||||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
|
||||
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
|
||||
}
|
||||
|
||||
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
|
||||
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
|
||||
// only for the rows kept.
|
||||
var gets, written, inFlight, maxFlight atomic.Int64
|
||||
vals := map[string]string{}
|
||||
for i := 0; i < 3000; i++ {
|
||||
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
|
||||
vals[k] = string(finished)
|
||||
if (i/100)%10 == 0 {
|
||||
vals[k] = string(building)
|
||||
}
|
||||
}
|
||||
slow := state(vals)
|
||||
get := slow.Get
|
||||
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
|
||||
gets.Add(1)
|
||||
n := inFlight.Add(1)
|
||||
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
inFlight.Add(-1)
|
||||
raw, _, err := get(k)
|
||||
return raw, time.Time{}, err
|
||||
}
|
||||
slow.Written = func(string) time.Time { written.Add(1); return at }
|
||||
for _, row := range []struct {
|
||||
module string
|
||||
stuck int
|
||||
gets int64
|
||||
cut bool
|
||||
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
|
||||
gets.Store(0)
|
||||
written.Store(0)
|
||||
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, cut := out["cut"]
|
||||
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
|
||||
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
|
||||
}
|
||||
}
|
||||
if maxFlight.Load() < 2 {
|
||||
t.Errorf("the values were read one at a time")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// heldAsks says an ask is held when its id is among held.
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
||||
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
|
||||
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
|
||||
// builder's state reasks, which the builder watches and answers there.
|
||||
const (
|
||||
StuckTool = "builder_stuck"
|
||||
AskAgainTool = "builder_ask_again"
|
||||
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||
ReAsksState = "reasks"
|
||||
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||
)
|
||||
|
||||
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
|
||||
// read it, its answer.
|
||||
type ReAsk struct {
|
||||
V int `json:"v"`
|
||||
Why string `json:"why"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// DecodeReAsk reads a reasks value.
|
||||
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||
var r ReAsk
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return r, err
|
||||
}
|
||||
if r.V != 1 {
|
||||
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
|
||||
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the build is asked again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, time.Time, error)
|
||||
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
|
||||
// only for a row they keep, and only when Get did not say.
|
||||
Written func(key string) time.Time
|
||||
}
|
||||
|
||||
// StuckAsk is one ask a key waits on.
|
||||
type StuckAsk struct {
|
||||
Language string `json:"language"`
|
||||
BuildModule string `json:"build_module"`
|
||||
ID string `json:"id"`
|
||||
Asking int `json:"asking"`
|
||||
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
|
||||
// state on its node since its start.
|
||||
Asked time.Time `json:"asked,omitzero"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
|
||||
// Stuck is one build key that waits on an ask.
|
||||
type Stuck struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
State string `json:"state"`
|
||||
Step string `json:"step"`
|
||||
By string `json:"by"`
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
Age string `json:"age,omitempty"`
|
||||
Asks []StuckAsk `json:"asks"`
|
||||
ReAsk *ReAsk `json:"reask,omitempty"`
|
||||
Unasked bool `json:"unasked,omitempty"`
|
||||
NoneStep int `json:"asks_with_no_step"`
|
||||
}
|
||||
|
||||
// StuckReader reads what builder_stuck answers.
|
||||
type StuckReader struct {
|
||||
Builds StateRead
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
|
||||
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
|
||||
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
|
||||
const (
|
||||
ReadWorkers = 16
|
||||
MaxReads = 1000
|
||||
)
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
defer func() {
|
||||
if r.Logf != nil {
|
||||
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
}()
|
||||
keys, err := r.Builds.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now
|
||||
if r.Now != nil {
|
||||
now = r.Now
|
||||
}
|
||||
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
|
||||
stepKeys := map[string]map[string]string{}
|
||||
for bm, s := range r.Steps {
|
||||
ks, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
|
||||
}
|
||||
byID := make(map[string]string, len(ks))
|
||||
for _, k := range ks {
|
||||
if _, id, err := delivery.ParseStepKey(k); err == nil {
|
||||
byID[id] = k
|
||||
}
|
||||
}
|
||||
stepKeys[bm] = byID
|
||||
}
|
||||
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
|
||||
// parallel, at most MaxReads of them.
|
||||
type parsed struct{ key, mod, fp string }
|
||||
var matching []parsed
|
||||
for _, key := range keys {
|
||||
mod, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||
if err != nil || (module != "" && mod != module) {
|
||||
continue
|
||||
}
|
||||
matching = append(matching, parsed{key, mod, fp})
|
||||
}
|
||||
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
|
||||
cut := len(matching) > MaxReads
|
||||
total := len(matching)
|
||||
if cut {
|
||||
matching = matching[:MaxReads]
|
||||
}
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
ats := make([]time.Time, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, at, err := r.Builds.Get(matching[i].key)
|
||||
if err == nil {
|
||||
values[i], ats[i] = raw, at
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reads += len(matching)
|
||||
out := []Stuck{}
|
||||
for i, m := range matching {
|
||||
key, mod, fp := m.key, m.mod, m.fp
|
||||
raw, at := values[i], ats[i]
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key)
|
||||
}
|
||||
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
|
||||
if !at.IsZero() {
|
||||
s.Age = now().Sub(at).Round(time.Second).String()
|
||||
}
|
||||
for language, id := range v.Asks {
|
||||
bm, _ := delivery.BuildModuleOf(language)
|
||||
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
|
||||
if t, ok := idTime(id); ok {
|
||||
a.Asked = t
|
||||
}
|
||||
if st, ok := r.Steps[bm]; ok {
|
||||
if k, ok := stepKeys[bm][id]; ok {
|
||||
reads++
|
||||
if raw, _, err := st.Get(k); err == nil && raw != nil {
|
||||
if step, err := delivery.DecodeStep(raw); err == nil {
|
||||
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.Step == "none" {
|
||||
s.NoneStep++
|
||||
}
|
||||
s.Asks = append(s.Asks, a)
|
||||
}
|
||||
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
|
||||
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
|
||||
if r.ReAsks.Get != nil {
|
||||
reads++
|
||||
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
|
||||
if ra, err := DecodeReAsk(raw); err == nil {
|
||||
s.ReAsk = &ra
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if !out[i].Since.Equal(out[j].Since) {
|
||||
return out[i].Since.Before(out[j].Since)
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// StuckBuildModules are the build modules whose steps builder_stuck reads.
|
||||
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
|
||||
|
||||
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
|
||||
func idTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
@@ -23,18 +23,21 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
@@ -84,13 +87,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
@@ -112,7 +115,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||
}
|
||||
@@ -127,7 +137,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
|
||||
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
@@ -149,8 +159,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
"module": "mesh-delivery-builder",
|
||||
"version": "1",
|
||||
"slug": "builder",
|
||||
"tools": [
|
||||
"builder_ask_again",
|
||||
"builder_seat_queue",
|
||||
"builder_stuck"
|
||||
],
|
||||
"emits": [
|
||||
"build.asked-again"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
@@ -12,7 +20,9 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "instances", "history": 1 }
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
@@ -21,6 +31,7 @@
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules"
|
||||
@@ -77,6 +88,17 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder",
|
||||
"binary": "mesh-delivery-builder"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder-tools",
|
||||
"binary": "mesh-delivery-builder-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-builder-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -15,8 +16,35 @@ import (
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
|
||||
var reads atomic.Int64
|
||||
|
||||
// cutAnswer says in the answer that it was cut, and how to narrow it.
|
||||
func cutAnswer(answer map[string]any, cut string) map[string]any {
|
||||
if cut != "" {
|
||||
answer["cut"], answer["note"] = true, cut
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
|
||||
func logged(tools []stdio.Tool) []stdio.Tool {
|
||||
for i := range tools {
|
||||
name, run := tools[i].Name, tools[i].Run
|
||||
tools[i].Run = func(args map[string]any) (any, error) {
|
||||
reads.Store(0)
|
||||
start := time.Now()
|
||||
out, err := run(args)
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) check.StateRead {
|
||||
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
reads.Add(1)
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
@@ -100,8 +128,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
group, _ := args["group"].(string)
|
||||
vs, err := r.Verdicts(group)
|
||||
return map[string]any{"verdicts": vs}, err
|
||||
vs, cut, err := r.Verdicts(group)
|
||||
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
@@ -122,8 +150,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
level, _ := args["level"].(string)
|
||||
digest, _ := args["digest"].(string)
|
||||
kept, err := r.Kept(level, digest)
|
||||
return map[string]any{"kept": kept}, err
|
||||
kept, cut, err := r.Kept(level, digest)
|
||||
return cutAnswer(map[string]any{"kept": kept}, cut), err
|
||||
}},
|
||||
{Name: check.PreviewsReadTool,
|
||||
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||
@@ -138,8 +166,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
if head != "" && len(head) < 7 {
|
||||
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||
}
|
||||
ps, bad, err := r.Previews(repository, head)
|
||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||
ps, bad, cut, err := r.Previews(repository, head)
|
||||
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
|
||||
}},
|
||||
{Name: check.StatusReadTool,
|
||||
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||
@@ -184,7 +212,7 @@ func main() {
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
||||
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -182,7 +182,8 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
|
||||
return b, err == nil, err
|
||||
}
|
||||
|
||||
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
|
||||
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
|
||||
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
|
||||
func (r *repoModules) CommitOf(module string) string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -152,17 +152,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
|
||||
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
|
||||
buckets := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
|
||||
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
|
||||
if buckets[name], err = bucket(name); err != nil {
|
||||
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -212,12 +207,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||
// says nothing of the builds, until the checker starts again.
|
||||
var builds check.BuildStates
|
||||
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
|
||||
default:
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
|
||||
builds = buildReader{kvStore{kv}}
|
||||
}
|
||||
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
|
||||
@@ -290,47 +280,38 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
|
||||
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
|
||||
// issue 1912), so the node-engine's have is not watched here.
|
||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
|
||||
default:
|
||||
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil {
|
||||
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
|
||||
// yet no trunk is known, so nothing is checked.
|
||||
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
default:
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil {
|
||||
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey {
|
||||
trunks.Put(e.Value(), !opOf(e))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
||||
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
|
||||
@@ -28,8 +28,11 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -38,14 +41,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -67,20 +70,31 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
open := !deleted && openAssignment(value)
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.standing[key] == open {
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = true
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -111,37 +125,58 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return false
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -155,17 +190,19 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,10 +3,19 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -44,10 +53,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -63,20 +72,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -88,25 +97,122 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -63,11 +65,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
||||
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
|
||||
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
|
||||
// through mesh-sdk (rules-are-weighed).
|
||||
const ReadWorkers = 16
|
||||
|
||||
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
|
||||
const MaxReads = 1000
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
|
||||
// further read starts.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
|
||||
func capKeys(keys []string, narrow string) ([]string, string) {
|
||||
if len(keys) <= MaxReads {
|
||||
return keys, ""
|
||||
}
|
||||
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
|
||||
}
|
||||
|
||||
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
|
||||
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
|
||||
func newestFirst(keys []string) {
|
||||
id := func(k string) (string, bool) {
|
||||
_, id, err := delivery.ParseStepKey(k)
|
||||
return id, err == nil
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a, aok := id(keys[i])
|
||||
b, bok := id(keys[j])
|
||||
switch {
|
||||
case aok != bok:
|
||||
return aok
|
||||
case len(a) != len(b):
|
||||
return len(a) > len(b)
|
||||
case a != b:
|
||||
return a > b
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
}
|
||||
@@ -89,37 +89,46 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
|
||||
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
|
||||
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
|
||||
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
|
||||
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
||||
return nil, fmt.Errorf("the verdicts state cannot be read here")
|
||||
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
|
||||
}
|
||||
keys, err := r.VerdictState.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []VerdictRead
|
||||
var matching []string
|
||||
for _, k := range keys {
|
||||
if group != "" {
|
||||
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
||||
continue
|
||||
}
|
||||
}
|
||||
raw, err := r.VerdictState.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
vr, err := r.read(k, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, vr)
|
||||
matching = append(matching, k)
|
||||
}
|
||||
return out, nil
|
||||
matching, cut = capKeys(matching, "give a group to narrow the answer")
|
||||
rows := make([]VerdictRead, len(matching))
|
||||
found := make([]bool, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, err := r.VerdictState.Get(matching[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return err
|
||||
}
|
||||
rows[i], err = r.read(matching[i], raw)
|
||||
found[i] = err == nil
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
for i, row := range rows {
|
||||
if found[i] {
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
return out, cut, nil
|
||||
}
|
||||
|
||||
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
||||
@@ -133,74 +142,95 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != repository || p.Number != number {
|
||||
keys, cut := capKeys(keys, "")
|
||||
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
||||
for from := 0; from < len(keys); from += ReadWorkers * 4 {
|
||||
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
|
||||
cands := make([]delivery.Candidate, len(batch))
|
||||
ok := make([]bool, len(batch))
|
||||
_ = parallel(len(batch), func(i int) error {
|
||||
raw, err := r.CandidateState.Get(batch[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
cands[i], ok[i] = c, err == nil
|
||||
return nil
|
||||
})
|
||||
for i, c := range cands {
|
||||
if !ok[i] {
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != repository || p.Number != number {
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
vraw, err := r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
if len(vraw) == 0 {
|
||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||
}
|
||||
return r.read(vkey, vraw)
|
||||
}
|
||||
vraw, err := r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
if len(vraw) == 0 {
|
||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||
}
|
||||
return r.read(vkey, vraw)
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
|
||||
}
|
||||
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
||||
}
|
||||
|
||||
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
|
||||
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
||||
return nil, fmt.Errorf("the kept state cannot be read here")
|
||||
return nil, "", fmt.Errorf("the kept state cannot be read here")
|
||||
}
|
||||
if digest != "" {
|
||||
key, err := delivery.KeptKey(level, digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
raw, err := r.KeptState.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no kept verdict %s", key)
|
||||
return nil, "", fmt.Errorf("no kept verdict %s", key)
|
||||
}
|
||||
return map[string]json.RawMessage{key: raw}, nil
|
||||
return map[string]json.RawMessage{key: raw}, "", nil
|
||||
}
|
||||
keys, err := r.KeptState.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := map[string]json.RawMessage{}
|
||||
var matching []string
|
||||
for _, k := range keys {
|
||||
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.KeptState.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out[k] = raw
|
||||
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
|
||||
matching = append(matching, k)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
matching, cut := capKeys(matching, "give a level and a digest to read one")
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
if err := parallel(len(matching), func(i int) (err error) {
|
||||
values[i], err = r.KeptState.Get(matching[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
for i, k := range matching {
|
||||
if len(values[i]) > 0 {
|
||||
out[k] = values[i]
|
||||
}
|
||||
}
|
||||
return out, cut, nil
|
||||
}
|
||||
|
||||
// PreviewRead is one preview as stored, with the parts of its key.
|
||||
@@ -214,17 +244,18 @@ type PreviewRead struct {
|
||||
|
||||
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
||||
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
|
||||
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
||||
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
||||
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
|
||||
}
|
||||
keys, err := r.PreviewState.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, "", err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []PreviewRead{}
|
||||
var bad []string
|
||||
type part struct{ repo, head, trunk string }
|
||||
parts := map[string]part{}
|
||||
var matching, bad []string
|
||||
for _, k := range keys {
|
||||
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
||||
if err != nil {
|
||||
@@ -234,15 +265,25 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, erro
|
||||
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.PreviewState.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
||||
parts[k] = part{repo, h, trunk}
|
||||
matching = append(matching, k)
|
||||
}
|
||||
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
if err := parallel(len(matching), func(i int) (err error) {
|
||||
values[i], err = r.PreviewState.Get(matching[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
out := []PreviewRead{}
|
||||
for i, k := range matching {
|
||||
if len(values[i]) > 0 {
|
||||
p := parts[k]
|
||||
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
|
||||
}
|
||||
}
|
||||
return out, bad, nil
|
||||
return out, bad, cut, nil
|
||||
}
|
||||
|
||||
// RunningStep is a step in flight on a build module.
|
||||
@@ -300,16 +341,28 @@ func (r Reader) Status() (Status, error) {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := sr.Get(k)
|
||||
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
||||
newestFirst(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
||||
}
|
||||
steps := make([]delivery.Step, len(keys))
|
||||
isRunning := make([]bool, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := sr.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
if err != nil || step.State != delivery.StepRunning {
|
||||
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
|
||||
return nil
|
||||
})
|
||||
for i := range keys {
|
||||
if !isRunning[i] {
|
||||
continue
|
||||
}
|
||||
step := steps[i]
|
||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||
if !step.Started.IsZero() {
|
||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||
@@ -344,23 +397,41 @@ func (r Reader) Status() (Status, error) {
|
||||
return st, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
total := len(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
|
||||
}
|
||||
type judged struct {
|
||||
c delivery.Candidate
|
||||
raw json.RawMessage
|
||||
ok bool
|
||||
}
|
||||
rows := make([]judged, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := r.CandidateState.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
rows[i].c = c
|
||||
if vraw, err := r.VerdictState.Get(vkey); err == nil {
|
||||
rows[i].raw, rows[i].ok = vraw, true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for _, row := range rows {
|
||||
if !row.ok {
|
||||
continue
|
||||
}
|
||||
raw, err = r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c, raw := row.c, row.raw
|
||||
if len(raw) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
|
||||
@@ -3,7 +3,9 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -51,7 +53,7 @@ func readerOf(t *testing.T) (Reader, string) {
|
||||
|
||||
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
|
||||
rd, key := readerOf(t)
|
||||
vs, err := rd.Verdicts("")
|
||||
vs, _, err := rd.Verdicts("")
|
||||
if err != nil || len(vs) != 1 || vs[0].Key != key {
|
||||
t.Fatalf("verdicts = %+v, %v", vs, err)
|
||||
}
|
||||
@@ -61,7 +63,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
|
||||
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
|
||||
t.Errorf("reports = %v", vs[0].Reports)
|
||||
}
|
||||
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
t.Errorf("another group's verdicts = %+v", only)
|
||||
}
|
||||
}
|
||||
@@ -85,11 +87,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
|
||||
|
||||
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
|
||||
rd, _ := readerOf(t)
|
||||
all, err := rd.Kept("", "")
|
||||
all, _, err := rd.Kept("", "")
|
||||
if err != nil || len(all) != 1 {
|
||||
t.Fatalf("kept = %v, %v", all, err)
|
||||
}
|
||||
one, err := rd.Kept("unit", digA)
|
||||
one, _, err := rd.Kept("unit", digA)
|
||||
if err != nil || len(one) != 1 {
|
||||
t.Fatalf("one kept = %v, %v", one, err)
|
||||
}
|
||||
@@ -98,10 +100,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
|
||||
t.Errorf("kept value = %+v, %v", k, err)
|
||||
}
|
||||
}
|
||||
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||
t.Errorf("integration = %v", none)
|
||||
}
|
||||
if _, err := rd.Kept("unit", digB); err == nil {
|
||||
if _, _, err := rd.Kept("unit", digB); err == nil {
|
||||
t.Error("a digest nothing kept was answered")
|
||||
}
|
||||
}
|
||||
@@ -115,17 +117,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
||||
}
|
||||
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
||||
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
||||
got, bad, err := rd.Previews(repoA, head1[:8])
|
||||
got, bad, _, err := rd.Previews(repoA, head1[:8])
|
||||
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
||||
t.Fatalf("previews = %+v, %v", got, err)
|
||||
}
|
||||
if len(bad) != 1 || bad[0] != "junk" {
|
||||
t.Errorf("unreadable = %v", bad)
|
||||
}
|
||||
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
t.Errorf("another repository's = %+v", got)
|
||||
}
|
||||
if _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
t.Error("answered with no state to read")
|
||||
}
|
||||
}
|
||||
@@ -233,3 +235,121 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||
type slowBucket struct {
|
||||
vals fakeState
|
||||
reads, lists atomic.Int64
|
||||
inFlight, maxFlight atomic.Int64
|
||||
}
|
||||
|
||||
func (b *slowBucket) get(key string) (json.RawMessage, error) {
|
||||
b.reads.Add(1)
|
||||
now := b.inFlight.Add(1)
|
||||
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
b.inFlight.Add(-1)
|
||||
return b.vals.read().Get(key)
|
||||
}
|
||||
|
||||
func (b *slowBucket) read() StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
|
||||
}
|
||||
|
||||
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
|
||||
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
|
||||
func (b *slowBucket) Create(string, []byte) error { return nil }
|
||||
|
||||
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
|
||||
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
|
||||
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
|
||||
kept := fakeState{}
|
||||
for i := 0; i < 1800; i++ {
|
||||
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
|
||||
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
|
||||
kept[k] = []byte(`{"v":1}`)
|
||||
}
|
||||
previews := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
|
||||
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
|
||||
previews[k] = []byte(`{"v":1}`)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
call func(*slowBucket) (rows int, cut string, err error)
|
||||
rows int
|
||||
reads, lists int64
|
||||
cut bool
|
||||
}{
|
||||
{"kept of one key", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
|
||||
return len(m), c, err
|
||||
}, 1, 1, 0, false},
|
||||
{"kept of a level", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
|
||||
return len(m), c, err
|
||||
}, 200, 200, 1, false},
|
||||
{"kept of everything", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
|
||||
return len(m), c, err
|
||||
}, MaxReads, MaxReads, 1, true},
|
||||
{"previews of a repository", func(b *slowBucket) (int, string, error) {
|
||||
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
|
||||
return len(p), c, err
|
||||
}, 1200, 1000, 1, true},
|
||||
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
|
||||
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
|
||||
return len(p), c, err
|
||||
}, 1, 1, 1, false},
|
||||
} {
|
||||
b := &slowBucket{}
|
||||
b.vals = kept
|
||||
if strings.HasPrefix(row.name, "previews") {
|
||||
b.vals = previews
|
||||
}
|
||||
start := time.Now()
|
||||
rows, cut, err := row.call(b)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
|
||||
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
|
||||
}
|
||||
if row.reads > 20 && b.maxFlight.Load() < 2 {
|
||||
t.Errorf("%s: values were read one at a time", row.name)
|
||||
}
|
||||
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
|
||||
}
|
||||
|
||||
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
|
||||
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
|
||||
steps := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
id := fmt.Sprintf("mvo5%04dabcdef", i)
|
||||
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
|
||||
if i == 1499 {
|
||||
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
|
||||
}
|
||||
raw, _ := delivery.EncodeStep(step)
|
||||
steps["g14."+id] = raw
|
||||
}
|
||||
sb := &slowBucket{vals: steps}
|
||||
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
Now: func() time.Time { return now }}.Status()
|
||||
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
|
||||
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
|
||||
}
|
||||
|
||||
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
|
||||
tries := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
|
||||
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
|
||||
}
|
||||
tb := &slowBucket{vals: tries}
|
||||
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
|
||||
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ const (
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
|
||||
MaxTries = 4
|
||||
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
|
||||
TryScan = 64
|
||||
// TryKeep is how long an ended try's key is kept before the checker removes it.
|
||||
TryKeep = 7 * 24 * time.Hour
|
||||
)
|
||||
@@ -155,15 +157,33 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var open []string
|
||||
for _, k := range keys {
|
||||
raw, err := store.Get(k)
|
||||
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the
|
||||
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last).
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(keys[i]) != len(keys[j]) {
|
||||
return len(keys[i]) > len(keys[j])
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
if len(keys) > TryScan {
|
||||
keys = keys[:TryScan]
|
||||
}
|
||||
tries := make([]Try, len(keys))
|
||||
live := make([]bool, len(keys))
|
||||
if err := parallel(len(keys), func(i int) error {
|
||||
raw, err := store.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
cur, err := DecodeTry(raw)
|
||||
if err != nil || cur.Over() {
|
||||
tries[i], live[i] = cur, err == nil && !cur.Over()
|
||||
return nil
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var open []string
|
||||
for i, cur := range tries {
|
||||
if !live[i] {
|
||||
continue
|
||||
}
|
||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
|
||||
@@ -22,6 +22,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -188,18 +190,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
call := callerOn(conn, cred.User)
|
||||
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
|
||||
var turn sync.Mutex
|
||||
kv, err := js.KeyValue(ctx, headsBucket)
|
||||
kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", headsBucket, err)
|
||||
return err
|
||||
}
|
||||
heads := kvHeads{kv}
|
||||
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
|
||||
rec := &recorder{pub: js, heads: heads, logger: logger}
|
||||
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
|
||||
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
|
||||
performedKV, err := js.KeyValue(ctx, performedBucket)
|
||||
performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
|
||||
return err
|
||||
}
|
||||
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
||||
watching := make(chan error, 3)
|
||||
@@ -207,16 +209,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
bucket string
|
||||
react func(ctx context.Context, key string, rev uint64, value []byte)
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
||||
kv, err := js.KeyValue(ctx, w.bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
// The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
|
||||
// by its delivery when the gate arrives.
|
||||
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
|
||||
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
|
||||
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
|
||||
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
|
||||
if kv == nil {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
|
||||
}
|
||||
go func() {
|
||||
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
@@ -231,7 +229,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
go onReconnect(ctx, again, said, logger)
|
||||
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
|
||||
// Essential: the forge's events are what the bridge carries.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
hear(&turn, func() {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -12,6 +13,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -66,6 +69,16 @@ func body(m map[string]any) []byte {
|
||||
}
|
||||
|
||||
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
||||
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||
var said []string
|
||||
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||
}
|
||||
})
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
@@ -230,3 +243,8 @@ func someFiles(_, _, path, _ string) ([]byte, error) {
|
||||
}
|
||||
return nil, bridge.ErrNotFound
|
||||
}
|
||||
|
||||
// refusingBus refuses every bucket with err (issue 1945).
|
||||
type refusingBus struct{ err error }
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -438,8 +440,9 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
|
||||
|
||||
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
|
||||
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
|
||||
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form
|
||||
// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature).
|
||||
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, its new form
|
||||
// (fingerprint or reason) does not move, and so the second writes no value at all (issue 1758, I9, ADR 0499 check clause 6
|
||||
// in miniature; the operator, 2026-10-25: a module's value moves only with the module).
|
||||
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
|
||||
if err != nil {
|
||||
@@ -448,7 +451,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
|
||||
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
|
||||
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
|
||||
merged := func(merge, root string) delivery.Module {
|
||||
publish := func(merge, root string, objects map[string]string) {
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
|
||||
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
|
||||
@@ -458,6 +461,8 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
at := func(merge string) delivery.Module {
|
||||
var m delivery.Module
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
|
||||
@@ -469,13 +474,35 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
|
||||
return m
|
||||
}
|
||||
first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
|
||||
second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
|
||||
if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||
t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2)
|
||||
const firstMerge, secondMerge, thirdMerge = "89abcdef0123456789abcdef0123456789abcdef", "99abcdef0123456789abcdef0123456789abcdef", "a9abcdef0123456789abcdef0123456789abcdef"
|
||||
publish(firstMerge, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", objects)
|
||||
first := at(firstMerge)
|
||||
if first.Inputs2 == nil || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||
t.Fatalf("kept inputs %+v, want a named tree", first.Inputs2)
|
||||
}
|
||||
if first.FP2 != second.FP2 || first.None2 != second.None2 {
|
||||
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
|
||||
// The go/-only merge, then one that changes src/: the stream is taken in order, so once the third is written the second
|
||||
// was heard, and it wrote nothing: the module's value stays at the commit its fingerprint first appeared at (the
|
||||
// operator, 2026-10-25).
|
||||
publish(secondMerge, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", objects)
|
||||
changed := maps.Clone(objects)
|
||||
changed["src"] = "6666666666666666666666666666666666666666"
|
||||
publish(thirdMerge, "cccccccccccccccccccccccccccccccccccccccc", changed)
|
||||
at(thirdMerge)
|
||||
history, err := modules.History(ctx, "mesh-sdk")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var commits []string
|
||||
for _, e := range history {
|
||||
if m, err := delivery.DecodeModule(e.Value()); err == nil {
|
||||
commits = append(commits, m.Commit)
|
||||
if m.Commit == firstMerge && (m.Inputs2 == nil || m.Inputs2.Tree != first.Inputs2.Tree || m.FP2 != first.FP2 || m.None2 != first.None2) {
|
||||
t.Errorf("the value at the first merge moved: %+v, was %+v", m, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(commits, secondMerge) {
|
||||
t.Errorf("the go/-only merge wrote a value: the commits written are %v", commits)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -615,17 +642,33 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||
return got
|
||||
}
|
||||
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
|
||||
if m := read(agreed); !agreed(m) {
|
||||
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", m.FP2, m.None2, m.Toolchains)
|
||||
held := read(agreed)
|
||||
if !agreed(held) {
|
||||
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
|
||||
}
|
||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
|
||||
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if e, err := modules.Get(ctx, "avahi"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
|
||||
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
|
||||
}
|
||||
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
|
||||
if moved == string(data) {
|
||||
t.Fatal("the merge names no tree to move")
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", []byte(moved)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disagree := func(m delivery.Module) bool {
|
||||
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
||||
}
|
||||
if m := read(disagree); !disagree(m) {
|
||||
t.Fatalf("within 5 s of n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||
}
|
||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -240,29 +240,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := bucket(delivery.BuildsBucket)
|
||||
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
|
||||
// such a module waits for its path, every other is indexed (issue 1945).
|
||||
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
|
||||
known := index.NewBuilds()
|
||||
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
|
||||
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
|
||||
@@ -273,12 +265,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
known.Delete(e.Key())
|
||||
}
|
||||
}
|
||||
w, err := settle(ctx, builds, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errs := make(chan error, 1)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
if builds != nil {
|
||||
if w, err := settle(ctx, builds, each); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
}
|
||||
}
|
||||
|
||||
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
|
||||
account, err := process.ReadAccount(path)
|
||||
@@ -296,22 +290,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
sourceErrs := make(chan error, 1)
|
||||
switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
default:
|
||||
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
|
||||
}
|
||||
}
|
||||
sw, err := settle(ctx, source, each)
|
||||
if err != nil {
|
||||
return err
|
||||
if sw, err := settle(ctx, source, each); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again")
|
||||
} else {
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
}
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
}
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
@@ -381,7 +370,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
|
||||
// Essential: the forge's merges are what the indexer indexes.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
|
||||
@@ -3,7 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -55,11 +55,14 @@ func (w *previewWatch) ensure() error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
|
||||
return nil
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
|
||||
return nil
|
||||
}
|
||||
w.watched = true
|
||||
go func() {
|
||||
|
||||
@@ -28,8 +28,11 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -38,14 +41,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -67,20 +70,31 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
open := !deleted && openAssignment(value)
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.standing[key] == open {
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = true
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -111,37 +125,58 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return false
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -155,17 +190,19 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -44,10 +45,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -63,20 +64,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -90,23 +91,75 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -73,11 +75,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -21,6 +22,7 @@ type laterBuckets struct {
|
||||
mu sync.Mutex
|
||||
appears map[string]jetstream.KeyValue // buckets that exist now
|
||||
asked map[string]int
|
||||
refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945)
|
||||
}
|
||||
|
||||
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
@@ -30,6 +32,9 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa
|
||||
if kv, ok := l.appears[name]; ok {
|
||||
return kv, nil
|
||||
}
|
||||
if l.refuse != nil {
|
||||
return nil, l.refuse
|
||||
}
|
||||
return nil, jetstream.ErrBucketNotFound
|
||||
}
|
||||
|
||||
@@ -100,6 +105,12 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
|
||||
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
|
||||
}
|
||||
var logs bytes.Buffer
|
||||
// 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running.
|
||||
refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")}
|
||||
if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String())
|
||||
}
|
||||
logs.Reset()
|
||||
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
|
||||
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -22,7 +22,8 @@ import (
|
||||
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
|
||||
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
|
||||
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
||||
// A value the module already holds is not written again.
|
||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
|
||||
@@ -85,14 +85,51 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
|
||||
|
||||
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
|
||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
rev, writes := mods.revs["a"], mods.writes
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
if mods.revs["a"] != rev || mods.writes != writes {
|
||||
t.Errorf("the same merge wrote again (%d writes, was %d)", mods.writes, writes)
|
||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing); the operator 2026-10-25 (a module's value moves only with the module)
|
||||
const repo, next = "novox/mesh-delivery", "cccccccccccccccccccccccccccccccccccccccc"
|
||||
three := map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
then func(ix *Indexer) error // after the first merge of a, b and c at merge, stamped by source revision 7
|
||||
// writes is the values written by then; commit is each module's commit after.
|
||||
writes int
|
||||
commit map[string]string
|
||||
}{
|
||||
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
|
||||
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
|
||||
then: func(ix *Indexer) error {
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||
then: func(ix *Indexer) error {
|
||||
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t)
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplySource(ctx, 7, sourceOf(repo, merge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writes := mods.writes
|
||||
if err := tc.then(ix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := mods.writes - writes; got != tc.writes {
|
||||
t.Errorf("%d values written, want %d", got, tc.writes)
|
||||
}
|
||||
for module, want := range tc.commit {
|
||||
if m, ok := moduleOf(t, mods, module); !ok || m.Commit != want {
|
||||
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,12 +112,12 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview
|
||||
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
|
||||
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
|
||||
func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) {
|
||||
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
trunk := ix.source.heads[repository]
|
||||
trunk := heads[repository]
|
||||
if trunk == "" {
|
||||
return delivery.Preview{}, false, false
|
||||
}
|
||||
@@ -137,14 +137,23 @@ func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, foun
|
||||
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
|
||||
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
|
||||
// no preview keeps what it had.
|
||||
//
|
||||
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
|
||||
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
|
||||
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
|
||||
// fingerprint first appeared at.
|
||||
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
|
||||
changed := false
|
||||
ix.mu.Lock()
|
||||
rev, heads := ix.source.rev, ix.source.heads
|
||||
ix.mu.Unlock()
|
||||
changed, read := false, false
|
||||
for i := range c.Pulls {
|
||||
p := &c.Pulls[i]
|
||||
pv, found, stale := ix.previewOf(p.Repository, p.Head.Head)
|
||||
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
|
||||
if !found && !stale {
|
||||
continue
|
||||
}
|
||||
read = true
|
||||
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
|
||||
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
|
||||
dirs := map[string]string{}
|
||||
@@ -198,6 +207,9 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
|
||||
if changed {
|
||||
c.FPSet = delivery.FingerprintSet(c.Pulls)
|
||||
}
|
||||
if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
|
||||
c.Base, changed = rev, true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -87,9 +89,16 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
|
||||
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
|
||||
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
|
||||
}
|
||||
if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
|
||||
c := candidateOf(t, cands)
|
||||
if c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
|
||||
t.Errorf("fpset = %q", c.FPSet)
|
||||
}
|
||||
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
|
||||
// the builder merges the preview onto that head, never onto the module value's commit.
|
||||
if c.Base != 5 {
|
||||
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
|
||||
}
|
||||
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
|
||||
// Another tree gives another fingerprint.
|
||||
pv := avahiPreview(t, trunkNow, h1)
|
||||
m := pv.Modules["modules/avahi"]
|
||||
@@ -191,3 +200,38 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
|
||||
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
|
||||
}
|
||||
}
|
||||
|
||||
// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the
|
||||
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
|
||||
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
|
||||
type writtenCandidate struct {
|
||||
Source sourceRev `json:"source"`
|
||||
Candidate delivery.Candidate `json:"candidate"`
|
||||
}
|
||||
|
||||
type sourceRev struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
}
|
||||
|
||||
func written(t *testing.T, w writtenCandidate) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(w, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = append(data, '\n')
|
||||
const file = "testdata/candidate-written.json"
|
||||
if os.Getenv("MESH_WRITE_TESTDATA") == "1" {
|
||||
if err := os.WriteFile(file, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(kept) != string(data) {
|
||||
t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,8 +32,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
|
||||
}
|
||||
|
||||
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
|
||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
|
||||
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
|
||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and
|
||||
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
|
||||
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
|
||||
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
|
||||
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
|
||||
@@ -71,7 +71,9 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
return nil, false, nil
|
||||
}
|
||||
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
|
||||
// Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another
|
||||
// repository) leaves the value as it is, so no revision of the source rewrites a module that did not change.
|
||||
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.Src = rev
|
||||
|
||||
@@ -38,15 +38,15 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
|
||||
if c, ok := moduleOf(t, mods, "c"); ok {
|
||||
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
|
||||
}
|
||||
// The same revision again changes nothing; a later revision moves Src.
|
||||
// The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit.
|
||||
if err := ix.ApplySource(ctx, 7, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplySource(ctx, 9, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 7 {
|
||||
t.Errorf("module a Src = %d, want 7", m.Src)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"source": {
|
||||
"rev": 5,
|
||||
"heads": {
|
||||
"novox/mesh-catalog": "5555555555555555555555555555555555555555"
|
||||
}
|
||||
},
|
||||
"candidate": {
|
||||
"v": 1,
|
||||
"group": "feat/x",
|
||||
"base": 5,
|
||||
"heads": "dfe9fb8f79e8",
|
||||
"pulls": [
|
||||
{
|
||||
"repository": "novox/mesh-catalog",
|
||||
"number": 301,
|
||||
"head": "0123456789abcdef0123456789abcdef01234567",
|
||||
"modules": [
|
||||
"avahi"
|
||||
],
|
||||
"fps": {
|
||||
"avahi": {
|
||||
"fp2": "src2:e8d595aea1b13f1f5578ef1f245dab0d28a351c28079ecdcd2276c63a932d7be",
|
||||
"manifest": "8a3856b6c0566d76f2556fc4f46c5251dc7935517bb6bd15809ded7e5546a0d8"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fpset": "01c840ec276b"
|
||||
}
|
||||
}
|
||||
@@ -112,6 +112,9 @@ type known struct {
|
||||
languages map[string]string
|
||||
bases []string
|
||||
contexts bool
|
||||
// held is the fingerprint the bucket holds for the module and the toolchains it was computed with (for a merge, the
|
||||
// value before it): what stillComputed keeps while the builders agree on no toolchain.
|
||||
held delivery.Module
|
||||
}
|
||||
|
||||
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
|
||||
@@ -159,20 +162,24 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
|
||||
all := map[string]*known{}
|
||||
for _, key := range keys {
|
||||
cur, ok := overlay[key]
|
||||
if !ok {
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if cur, err = delivery.DecodeModule(value); err != nil {
|
||||
cur, merged := overlay[key]
|
||||
var before delivery.Module
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
if found {
|
||||
if before, err = delivery.DecodeModule(value); err != nil && !merged {
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !merged {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
cur = before
|
||||
}
|
||||
held[key] = cur
|
||||
if cur.Manifest == "" || cur.Inputs2 == nil {
|
||||
continue
|
||||
@@ -186,7 +193,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
cur, stale[key] = again, true
|
||||
held[key] = cur
|
||||
}
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||
}
|
||||
tools := ix.agreed()
|
||||
type result struct {
|
||||
@@ -251,6 +258,9 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
r.none = strings.Join(missing, "; ")
|
||||
case noToolchain != "":
|
||||
r.none = noToolchain
|
||||
if fp, ok := stillComputed(module, kept, k, toolchains); ok {
|
||||
r = result{fp: fp, tcs: k.held.Toolchains}
|
||||
}
|
||||
default:
|
||||
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||
if r.fp != "" {
|
||||
@@ -309,6 +319,12 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
|
||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||
// the ones that were computed, the others are written as the merge gave them.
|
||||
//
|
||||
// **A module the merge did not change is not written** (the operator, 2026-10-25: "only if a module has got a new version in
|
||||
// commit x, should a new module version get created linked to commit x"). A value names the commit its fingerprint first
|
||||
// appeared at: when the merge gives a module the value it holds but for the commit (and the source revision stamped on it),
|
||||
// the held value stands, its commit unchanged, so a catalogue merge touching one module writes one key, not one per module of
|
||||
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
|
||||
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
|
||||
for _, key := range slices.Sorted(maps.Keys(overlay)) {
|
||||
next := overlay[key]
|
||||
@@ -317,7 +333,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
}
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && sameButWhere(cur, next) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
@@ -331,6 +347,13 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
return nil
|
||||
}
|
||||
|
||||
// sameButWhere says the held value cur is next but for where it was computed (Commit and Src): the module is unchanged.
|
||||
func sameButWhere(cur, next delivery.Module) bool {
|
||||
next = withV(next)
|
||||
next.Commit, next.Src = cur.Commit, cur.Src
|
||||
return reflect.DeepEqual(cur, next)
|
||||
}
|
||||
|
||||
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
|
||||
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
@@ -381,6 +404,32 @@ func hasContext(m catalogue.Manifest) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// stillComputed is the fingerprint a module already holds, kept while a language it is built with has no agreed toolchain
|
||||
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
|
||||
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
|
||||
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
|
||||
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
|
||||
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
|
||||
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2) (string, bool) {
|
||||
if k.held.FP2 == "" {
|
||||
return "", false
|
||||
}
|
||||
toolchains := map[string]delivery.Toolchain2{}
|
||||
for artifact, language := range k.languages {
|
||||
if tc, ok := agreed[artifact]; ok {
|
||||
toolchains[artifact] = tc
|
||||
continue
|
||||
}
|
||||
used, ok := k.held.Toolchains[language]
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||
}
|
||||
fp, none := delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||
return fp, none == "" && fp == k.held.FP2
|
||||
}
|
||||
|
||||
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
||||
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
|
||||
if len(by) == 0 {
|
||||
|
||||
@@ -123,9 +123,16 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
|
||||
}
|
||||
|
||||
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
|
||||
// Both reports are read before the module's first fingerprint is computed (a fingerprint computed before they
|
||||
// disagree is kept: TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith).
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||
for node, version := range map[string]string{"novox": "go1.27.1", "ace": "go1.26.0"} {
|
||||
key, _ := delivery.ToolchainKey(node, "go")
|
||||
ix.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", version, "1"), false)
|
||||
}
|
||||
if err := ix.Recompute(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
|
||||
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
|
||||
@@ -260,11 +267,13 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
|
||||
w := mods.writes
|
||||
// the same catalogue merged again at a later commit: nothing moved
|
||||
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
f.Merge = otherMerge
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w {
|
||||
t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w)
|
||||
// and the commit stays the one the fingerprint first appeared at (the operator, 2026-10-25)
|
||||
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || after.Commit != before.Commit || mods.writes != w {
|
||||
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.FP2, after.Commit, before.FP2, before.Commit, mods.writes-w)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,9 +297,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
|
||||
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
|
||||
}
|
||||
// Nodes that disagree leave no fingerprint and no toolchains.
|
||||
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
|
||||
// stale report blanked every module and the controller's take-in refused modules already built); only a new
|
||||
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil {
|
||||
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v, want neither", d.FP2, d.Toolchains)
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != n.FP2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
|
||||
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.FP2, d.Toolchains, d.None2, n.FP2, n.Toolchains)
|
||||
}
|
||||
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
f.Trees["modules/avahi"] = "tree-avahi-moved"
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
|
||||
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.FP2, d.Toolchains, d.None2)
|
||||
}
|
||||
// They agree again: the new tree gets its fingerprint.
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.2")
|
||||
if a, _ := moduleOf(t, mods, "avahi"); a.FP2 == "" || a.FP2 == n.FP2 {
|
||||
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.FP2, n.FP2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -119,12 +121,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err)
|
||||
// Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant).
|
||||
return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err)
|
||||
}
|
||||
// The answers it writes and what it asked: its own work, so each stops the start, naming its grant.
|
||||
var kvs [2]jetstream.KeyValue
|
||||
for i, name := range []string{delivery.AnswersBucket, askedBucket} {
|
||||
if kvs[i], err = js.KeyValue(ctx, name); err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
state := kvStore{ctx: ctx, kv: kvs[1]}
|
||||
@@ -140,7 +144,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
|
||||
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
||||
failed := make(chan error, 2)
|
||||
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
||||
assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf)
|
||||
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
||||
if viewKV != nil {
|
||||
t.View = kvGate{ctx: ctx, kv: viewKV}
|
||||
@@ -172,15 +176,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
|
||||
// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not.
|
||||
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
|
||||
return nil
|
||||
}
|
||||
return kv
|
||||
}
|
||||
// teller is what an optional bucket of the teller leaves out.
|
||||
const teller = "the asks and answers it holds are not carried until the teller is started again with it"
|
||||
|
||||
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
|
||||
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
@@ -236,6 +241,16 @@ func (b *bed) askedOf(subject string) []published {
|
||||
}
|
||||
|
||||
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
|
||||
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||
var said []string
|
||||
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||
}
|
||||
})
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
@@ -518,3 +533,8 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R
|
||||
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus refuses every bucket with err (issue 1945).
|
||||
type refusingBus struct{ err error }
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -9,8 +9,9 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"mesh-delivery-view/internal/view"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Tools are the read tools over a modules state.
|
||||
@@ -72,7 +73,7 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
}},
|
||||
{Name: view.BuilderBuildReadTool,
|
||||
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
|
||||
"at the fingerprint the indexer holds for it now, joined through modules.<module>; with commit, only if the indexer holds the module at that commit. Read-only.",
|
||||
"at the fingerprint the indexer holds for it now, joined through modules.<module>; with commit, only if the indexer holds the module at that commit (a module's value names the commit its fingerprint first appeared at, and stays there through merges that leave the module alone). Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp2": map[string]any{"type": "string", "description": "its new-form fingerprint; empty means the indexer's current one"},
|
||||
@@ -85,7 +86,8 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
}},
|
||||
{Name: view.WantsReadTool,
|
||||
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
|
||||
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
|
||||
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given); a node and a module read that one key. " +
|
||||
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
@@ -100,7 +102,8 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
}},
|
||||
{Name: view.DifferencesReadTool,
|
||||
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
|
||||
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
|
||||
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module (both: that one key). " +
|
||||
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
@@ -108,7 +111,8 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
}},
|
||||
{Name: view.HavesReadTool,
|
||||
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
|
||||
"and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
|
||||
"and <node>.node (the node's own facts). Narrow by node and module (both: that one key). " +
|
||||
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
)
|
||||
@@ -164,7 +166,7 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
|
||||
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
|
||||
type bigBuilds struct {
|
||||
fake fake
|
||||
reads *int
|
||||
reads *atomic.Int64
|
||||
}
|
||||
|
||||
func newBig(n int) bigBuilds {
|
||||
@@ -172,15 +174,15 @@ func newBig(n int) bigBuilds {
|
||||
for i := 0; i < n; i++ {
|
||||
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
|
||||
}
|
||||
return bigBuilds{fake: f, reads: new(int)}
|
||||
return bigBuilds{fake: f, reads: new(atomic.Int64)}
|
||||
}
|
||||
|
||||
func (b bigBuilds) reader() view.Reader {
|
||||
r := b.fake.reader()
|
||||
get := r.State.Get
|
||||
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
|
||||
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { b.reads.Add(1); return get(k) },
|
||||
GetEntry: func(k string) (json.RawMessage, uint64, error) {
|
||||
*b.reads++
|
||||
b.reads.Add(1)
|
||||
raw, err := get(k)
|
||||
var rev uint64
|
||||
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
|
||||
@@ -198,8 +200,8 @@ func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(view.BuildsAnswer)
|
||||
if *b.reads != 20 {
|
||||
t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
|
||||
if b.reads.Load() != 20 {
|
||||
t.Errorf("read %d values for a module of 20 keys, want 20", b.reads.Load())
|
||||
}
|
||||
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
|
||||
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
|
||||
@@ -222,13 +224,13 @@ func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(view.BuildsAnswer)
|
||||
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
|
||||
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
|
||||
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || b.reads.Load() > view.DefaultBuildsLimit {
|
||||
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, b.reads.Load())
|
||||
}
|
||||
b = newBig(2000)
|
||||
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
|
||||
if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
|
||||
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
|
||||
if a := out.(view.BuildsAnswer); b.reads.Load() != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
|
||||
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", b.reads.Load(), view.MaxBuildReads, a.Cut)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,6 +282,41 @@ func keptReader() view.Reader {
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket is a bucket of the live shape (2026-10-25: node_haves and assigner_wants timed out at 30 s): keys over 4 nodes
|
||||
// and perNode modules each, written by key(node, module), whose Get takes delay and counts the values read, the lists of
|
||||
// the bucket and the most Gets in flight at once.
|
||||
type slowBucket struct {
|
||||
fake fake
|
||||
delay time.Duration
|
||||
reads, lists atomic.Int64
|
||||
inFlight, maxFlight atomic.Int64
|
||||
}
|
||||
|
||||
func newSlow(perNode int, delay time.Duration, key func(node string, i int) string) *slowBucket {
|
||||
b := &slowBucket{fake: fake{}, delay: delay}
|
||||
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
|
||||
for i := 0; i < perNode; i++ {
|
||||
b.fake[key(n, i)] = `{"fp":"x"}`
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *slowBucket) state() view.StateRead {
|
||||
f := b.fake.reader().State
|
||||
return view.StateRead{
|
||||
Keys: func() ([]string, error) { b.lists.Add(1); return f.Keys() },
|
||||
Get: func(k string) (json.RawMessage, error) {
|
||||
b.reads.Add(1)
|
||||
now := b.inFlight.Add(1)
|
||||
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(b.delay)
|
||||
b.inFlight.Add(-1)
|
||||
return f.Get(k)
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.WantsReadTool, map[string]any{})
|
||||
@@ -298,6 +335,39 @@ func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
||||
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
||||
t.Errorf("module zsh = %v", got)
|
||||
}
|
||||
|
||||
// On the live shape (4 nodes, 150 wants each, 1 ms a Get): a node and a module read that one key and list nothing;
|
||||
// a node reads only its own keys; the whole answer reads its values in parallel.
|
||||
slow := newSlow(150, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("want.%s.m%03d", n, i) })
|
||||
r = view.Reader{WantsState: slow.state()}
|
||||
for _, row := range []struct {
|
||||
args map[string]any
|
||||
wants int
|
||||
reads, lists int64
|
||||
}{
|
||||
{map[string]any{"node": "ace", "module": "m007"}, 1, 1, 0},
|
||||
{map[string]any{"node": "ace", "module": "gone"}, 0, 1, 0},
|
||||
{map[string]any{"node": "g14"}, 150, 150, 1},
|
||||
{map[string]any{"module": "m007"}, 4, 4, 1},
|
||||
{map[string]any{}, 600, 600, 1},
|
||||
} {
|
||||
slow.reads.Store(0)
|
||||
slow.lists.Store(0)
|
||||
start := time.Now()
|
||||
out, err := run(t, r, view.WantsReadTool, row.args)
|
||||
took := time.Since(start)
|
||||
if err != nil {
|
||||
t.Fatalf("%v: %v", row.args, err)
|
||||
}
|
||||
a := out.(view.WantsAnswer)
|
||||
if len(a.Wants) != row.wants || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut {
|
||||
t.Errorf("%v: %d wants, %d reads, %d lists, cut %v; want %d, %d, %d", row.args, len(a.Wants), slow.reads.Load(), slow.lists.Load(), a.Cut, row.wants, row.reads, row.lists)
|
||||
}
|
||||
t.Logf("assigner_wants %v: %d values in %v (one after the other: %v)", row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
|
||||
}
|
||||
if slow.maxFlight.Load() < 2 {
|
||||
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
|
||||
@@ -328,4 +398,39 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
|
||||
t.Errorf("haves of ace (the node's own key too) = %v", got)
|
||||
}
|
||||
|
||||
// More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list
|
||||
// nothing; the whole answer reads MaxStateReads values in parallel and says it was cut.
|
||||
slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) })
|
||||
r = view.Reader{HavesState: slow.state(), DifferencesState: slow.state()}
|
||||
for _, row := range []struct {
|
||||
tool string
|
||||
args map[string]any
|
||||
keys int
|
||||
reads, lists int64
|
||||
cut bool
|
||||
}{
|
||||
{view.HavesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
|
||||
{view.DifferencesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
|
||||
{view.HavesReadTool, map[string]any{"node": "ace"}, 300, 300, 1, false},
|
||||
{view.HavesReadTool, map[string]any{"module": "m042"}, 4, 4, 1, false},
|
||||
{view.HavesReadTool, map[string]any{}, view.MaxStateReads, view.MaxStateReads, 1, true},
|
||||
} {
|
||||
slow.reads.Store(0)
|
||||
slow.lists.Store(0)
|
||||
start := time.Now()
|
||||
out, err := run(t, r, row.tool, row.args)
|
||||
took := time.Since(start)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", row.tool, row.args, err)
|
||||
}
|
||||
a := out.(view.KeptAnswer)
|
||||
if len(a.Keys) != row.keys || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut != row.cut || (a.Cut && !strings.Contains(a.Note, "narrow")) {
|
||||
t.Errorf("%s %v: %d keys, %d reads, %d lists, cut %v %q; want %d, %d, %d, %v", row.tool, row.args, len(a.Keys), slow.reads.Load(), slow.lists.Load(), a.Cut, a.Note, row.keys, row.reads, row.lists, row.cut)
|
||||
}
|
||||
t.Logf("%s %v: %d values in %v (one after the other: %v)", row.tool, row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
|
||||
}
|
||||
if slow.maxFlight.Load() < 2 {
|
||||
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81 h1:LhljEtt7FbNpVn67eOQbdsFDS8Qj6WPUF6l9IXr9mo4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -70,12 +72,13 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
values, err := r.State.values(keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
raw, err := r.State.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, k := range keys {
|
||||
raw := values[i]
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -129,13 +132,14 @@ func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string,
|
||||
return nil, nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
values, err := r.CandidatesState.values(keys)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
var unreadable []string
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidatesState.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for i, k := range keys {
|
||||
raw := values[i]
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -242,35 +246,51 @@ func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, er
|
||||
}
|
||||
candidates = append(candidates, k)
|
||||
}
|
||||
// The values are read in batches, each batch in parallel and taken in key order: without a module or a state a batch
|
||||
// is only the rows still missing, so no more is read than the answer needs; otherwise up to readWorkers*4.
|
||||
reads := 0
|
||||
for _, k := range candidates {
|
||||
walk:
|
||||
for reads < len(candidates) {
|
||||
if reads >= MaxBuildReads {
|
||||
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
|
||||
break
|
||||
}
|
||||
reads++
|
||||
row, ok, err := r.builderRow(k)
|
||||
if err != nil {
|
||||
size := readWorkers * 4
|
||||
if prefix == "" && state == "" {
|
||||
size = limit - len(out.Builds)
|
||||
}
|
||||
size = min(size, MaxBuildReads-reads, len(candidates)-reads)
|
||||
batch := candidates[reads : reads+size]
|
||||
rows := make([]BuilderRow, len(batch))
|
||||
found := make([]bool, len(batch))
|
||||
if err := parallel(len(batch), func(i int) error {
|
||||
var err error
|
||||
rows[i], found[i], err = r.builderRow(batch[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return BuildsAnswer{}, err
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if state != "" {
|
||||
var b struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
|
||||
reads += len(batch)
|
||||
for i, row := range rows {
|
||||
if !found[i] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out.Builds = append(out.Builds, row)
|
||||
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
||||
if prefix == "" && len(out.Builds) >= limit {
|
||||
if reads < len(candidates) {
|
||||
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
||||
if state != "" {
|
||||
var b struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out.Builds = append(out.Builds, row)
|
||||
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
||||
if prefix == "" && len(out.Builds) >= limit {
|
||||
if reads-len(batch)+i+1 < len(candidates) {
|
||||
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
||||
}
|
||||
break walk
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out.Builds, func(i, j int) bool {
|
||||
@@ -311,7 +331,10 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
||||
return BuilderRow{}, fmt.Errorf("the indexer's value of %s is not readable: %w", module, err)
|
||||
}
|
||||
if commit != "" && m.Commit != commit {
|
||||
return BuilderRow{}, fmt.Errorf("the indexer holds %s at commit %s, not %s, so its fingerprint is not the one of that commit", module, m.Commit, commit)
|
||||
// The value names the commit its fingerprint first appeared at, and a later merge that left the module alone
|
||||
// writes none (the operator, 2026-10-25): another commit may well have the same fingerprint, which is not guessed.
|
||||
return BuilderRow{}, fmt.Errorf("the indexer holds %s at commit %s, the commit its fingerprint first appeared at, not %s; "+
|
||||
"a later trunk commit that left the module alone keeps that fingerprint: ask without commit for the build at it", module, m.Commit, commit)
|
||||
}
|
||||
if m.FP2 == "" {
|
||||
return BuilderRow{}, fmt.Errorf("the indexer holds no new-form fingerprint of %s: %s", module, m.None2)
|
||||
@@ -349,6 +372,9 @@ type WantsAnswer struct {
|
||||
Wants map[string]json.RawMessage `json:"wants"`
|
||||
Held map[string]json.RawMessage `json:"held"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
// Cut says the answer stopped at MaxStateReads values, and Note how to narrow it.
|
||||
Cut bool `json:"cut,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
|
||||
@@ -356,60 +382,158 @@ type WantsAnswer struct {
|
||||
type KeptAnswer struct {
|
||||
Keys map[string]json.RawMessage `json:"keys"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
Cut bool `json:"cut,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
|
||||
if s.Keys == nil || s.Get == nil {
|
||||
return nil, fmt.Errorf("%s cannot be read here", what)
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// readWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
||||
// runtime to the bus (the runtime opens the bucket and gets the key: two bus requests), so reading a bucket of a few
|
||||
// hundred keys one after the other outlasted the 30 s a tool call is given on 2026-10-25 (node_haves, assigner_wants).
|
||||
const readWorkers = 16
|
||||
|
||||
// MaxStateReads bounds the values one call of assigner_wants, assigner_differences or node_haves reads; past it the
|
||||
// answer says it was cut and how to narrow it.
|
||||
const MaxStateReads = 1000
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most readWorkers at a time, and answers the first error; once one failed no
|
||||
// further read starts.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
workers := min(readWorkers, n)
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// values reads the stored value of each key, in parallel, in the order of keys (nil for a key with none).
|
||||
func (s StateRead) values(keys []string) ([]json.RawMessage, error) {
|
||||
out := make([]json.RawMessage, len(keys))
|
||||
err := parallel(len(keys), func(i int) error {
|
||||
raw, err := s.Get(keys[i])
|
||||
out[i] = raw
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// read reads the values of keys (sorted first) into a map, at most MaxStateReads of them; cut says it stopped there.
|
||||
func (s StateRead) read(keys []string) (map[string]json.RawMessage, bool, error) {
|
||||
sort.Strings(keys)
|
||||
cut := len(keys) > MaxStateReads
|
||||
if cut {
|
||||
keys = keys[:MaxStateReads]
|
||||
}
|
||||
values, err := s.values(keys)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
raw, err := s.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out[k] = raw
|
||||
for i, k := range keys {
|
||||
if len(values[i]) > 0 {
|
||||
out[k] = values[i]
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
return out, cut, nil
|
||||
}
|
||||
|
||||
func cutNote(cut bool, n int) string {
|
||||
if !cut {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("read the first %d of %d matching keys in key order and stopped; give a node and a module to narrow the answer", MaxStateReads, n)
|
||||
}
|
||||
|
||||
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
|
||||
// matches).
|
||||
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
|
||||
all, err := r.WantsState.all("the assigner's wants")
|
||||
s := r.WantsState
|
||||
if s.Get == nil {
|
||||
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
|
||||
}
|
||||
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
|
||||
if node != "" && module != "" {
|
||||
// One key, read directly: no list of the bucket (a hold has no node, so none is shown).
|
||||
key, err := delivery.WantKey(node, module)
|
||||
if module == "node" {
|
||||
key, err = delivery.WantNodeKey(node)
|
||||
}
|
||||
if err != nil {
|
||||
return WantsAnswer{}, err
|
||||
}
|
||||
raw, err := s.Get(key)
|
||||
if err != nil {
|
||||
return WantsAnswer{}, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out.Wants[key] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if s.Keys == nil {
|
||||
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return WantsAnswer{}, err
|
||||
}
|
||||
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
|
||||
for k, raw := range all {
|
||||
// Narrowed by the key before any value is read.
|
||||
var matching []string
|
||||
for _, k := range keys {
|
||||
if n, m, err := delivery.ParseWantKey(k); err == nil {
|
||||
if (node == "" || n == node) && (module == "" || m == module) {
|
||||
out.Wants[k] = raw
|
||||
matching = append(matching, k)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
|
||||
if n, err := delivery.ParseWantNodeKey(k); err == nil {
|
||||
if (node == "" || n == node) && (module == "" || module == "node") {
|
||||
out.Wants[k] = raw
|
||||
matching = append(matching, k)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if m, _, err := delivery.ParseHeldKey(k); err == nil {
|
||||
if node == "" && (module == "" || m == module) {
|
||||
out.Held[k] = raw
|
||||
matching = append(matching, k)
|
||||
}
|
||||
continue
|
||||
}
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
}
|
||||
sort.Strings(out.Unreadable)
|
||||
values, cut, err := s.read(matching)
|
||||
if err != nil {
|
||||
return WantsAnswer{}, err
|
||||
}
|
||||
out.Cut, out.Note = cut, cutNote(cut, len(matching))
|
||||
for k, raw := range values {
|
||||
if _, _, err := delivery.ParseHeldKey(k); err == nil {
|
||||
out.Held[k] = raw
|
||||
} else {
|
||||
out.Wants[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -433,21 +557,50 @@ func (r Reader) Want(node, module string) (json.RawMessage, error) {
|
||||
}
|
||||
|
||||
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
|
||||
all, err := s.all(what)
|
||||
if s.Get == nil {
|
||||
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
|
||||
}
|
||||
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
|
||||
if node != "" && module != "" {
|
||||
// One key, read directly: no list of the bucket.
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return KeptAnswer{}, err
|
||||
}
|
||||
raw, err := s.Get(key)
|
||||
if err != nil {
|
||||
return KeptAnswer{}, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out.Keys[key] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if s.Keys == nil {
|
||||
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return KeptAnswer{}, err
|
||||
}
|
||||
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
|
||||
for k, raw := range all {
|
||||
// Narrowed by the key before any value is read.
|
||||
var matching []string
|
||||
for _, k := range keys {
|
||||
n, m, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil {
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
continue
|
||||
}
|
||||
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
|
||||
out.Keys[k] = raw
|
||||
matching = append(matching, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.Unreadable)
|
||||
values, cut, err := s.read(matching)
|
||||
if err != nil {
|
||||
return KeptAnswer{}, err
|
||||
}
|
||||
out.Keys, out.Cut, out.Note = values, cut, cutNote(cut, len(matching))
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -222,23 +224,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
// Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants
|
||||
// and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build
|
||||
// seats' steps, and says that moves are not watched (issue 1945).
|
||||
var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue }
|
||||
for _, b := range []struct {
|
||||
name string
|
||||
into *jetstream.KeyValue
|
||||
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck},
|
||||
{delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} {
|
||||
if *b.into, err = bucket(b.name); err != nil {
|
||||
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} {
|
||||
if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
kvs.wants = process.OptionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf)
|
||||
kvs.haves = process.OptionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf)
|
||||
if kvs.wants == nil || kvs.haves == nil {
|
||||
kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it
|
||||
}
|
||||
|
||||
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck}, time.Now)
|
||||
w.Logf(logger.Printf)
|
||||
@@ -283,13 +285,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent
|
||||
// until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse.
|
||||
haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
if err != nil {
|
||||
return err
|
||||
var haveSeen, wantSeen jetstream.KeyWatcher
|
||||
if kvs.haves != nil {
|
||||
if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.HaveBucket, err, "moves are not watched until the watchman starts again")
|
||||
} else if wantSeen, err = settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again")
|
||||
haveSeen.Stop()
|
||||
haveSeen = nil
|
||||
}
|
||||
}
|
||||
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
|
||||
// keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same.
|
||||
@@ -304,7 +308,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
lang := lang
|
||||
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) })
|
||||
if err != nil {
|
||||
return err
|
||||
logger.Printf("the steps of %s cannot be watched (%v): its asks are mirrored and never seen taken", module, err)
|
||||
continue
|
||||
}
|
||||
taken[lang] = sw
|
||||
}
|
||||
@@ -348,12 +353,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
go func() {
|
||||
errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
}()
|
||||
if haveSeen != nil && wantSeen != nil {
|
||||
go func() {
|
||||
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
}()
|
||||
}
|
||||
for lang, sw := range taken {
|
||||
lang, sw := lang, sw
|
||||
go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }()
|
||||
|
||||
@@ -2,11 +2,17 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6
|
||||
@@ -27,8 +33,23 @@ func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
|
||||
func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||
var said []string
|
||||
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "watchman", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||
}
|
||||
})
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus refuses every bucket with err (issue 1945).
|
||||
type refusingBus struct{ err error }
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// requestWithin bounds one request to the bus: a read the bus refuses is answered by no reply, so the bound is how a
|
||||
// refusal is told from a slow answer. It is no clock of the part's work: nothing is ever started by it.
|
||||
const requestWithin = 10 * time.Second
|
||||
|
||||
// bucketOf is the bucket a read names: `<module>.<state>` is `<module>_<state>`.
|
||||
func bucketOf(state string) string {
|
||||
at := strings.LastIndex(state, ".")
|
||||
if at <= 0 {
|
||||
return state
|
||||
}
|
||||
return state[:at] + "_" + state[at+1:]
|
||||
}
|
||||
|
||||
// busReader reads other parts' state on the bus, and keeps every value of a state it watches, so a composition reads
|
||||
// those from memory. A bucket it cannot open (refused, or not there yet) is an error of the call, never the end.
|
||||
type busReader struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
|
||||
mu sync.Mutex
|
||||
kvs map[string]jetstream.KeyValue
|
||||
world map[string]map[string][]byte
|
||||
settled map[string]bool
|
||||
}
|
||||
|
||||
func newBusReader(ctx context.Context, js jetstream.JetStream) *busReader {
|
||||
return &busReader{ctx: ctx, js: js, kvs: map[string]jetstream.KeyValue{}, world: map[string]map[string][]byte{},
|
||||
settled: map[string]bool{}}
|
||||
}
|
||||
|
||||
func (b *busReader) kv(state string) (jetstream.KeyValue, error) {
|
||||
b.mu.Lock()
|
||||
kv, ok := b.kvs[state]
|
||||
b.mu.Unlock()
|
||||
if ok {
|
||||
return kv, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(b.ctx, requestWithin)
|
||||
defer cancel()
|
||||
kv, err := b.js.KeyValue(ctx, bucketOf(state))
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = fmt.Errorf("no answer in %s: a read the bus refuses is answered by none (%w)", requestWithin, err)
|
||||
}
|
||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", bucketOf(state), err)
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.kvs[state] = kv
|
||||
b.mu.Unlock()
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
func (b *busReader) Keys(state string) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
if b.settled[state] {
|
||||
out := make([]string, 0, len(b.world[state]))
|
||||
for k := range b.world[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
b.mu.Unlock()
|
||||
kv, err := b.kv(state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(b.ctx, requestWithin)
|
||||
defer cancel()
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return []string{}, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (b *busReader) Get(state, key string) ([]byte, bool, error) {
|
||||
b.mu.Lock()
|
||||
if b.settled[state] {
|
||||
v, ok := b.world[state][key]
|
||||
b.mu.Unlock()
|
||||
return v, ok, nil
|
||||
}
|
||||
b.mu.Unlock()
|
||||
kv, err := b.kv(state)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(b.ctx, requestWithin)
|
||||
defer cancel()
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
|
||||
// Watch keeps every value of a state from now on and hands each change after what stands to changed; it returns once
|
||||
// what stands is held. The watch ending is handed to ended, and the state is read from the bus again until watched.
|
||||
func (b *busReader) Watch(state string, changed func(string, []byte), ended func(error)) error {
|
||||
kv, err := b.kv(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := kv.WatchAll(b.ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched: %w", bucketOf(state), err)
|
||||
}
|
||||
put := func(e jetstream.KeyValueEntry) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.world[state] == nil {
|
||||
b.world[state] = map[string][]byte{}
|
||||
}
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
b.world[state][e.Key()] = e.Value()
|
||||
} else {
|
||||
delete(b.world[state], e.Key())
|
||||
}
|
||||
}
|
||||
lost := func(err error) {
|
||||
w.Stop()
|
||||
b.mu.Lock()
|
||||
b.settled[state] = false
|
||||
delete(b.world, state)
|
||||
delete(b.kvs, state)
|
||||
b.mu.Unlock()
|
||||
ended(err)
|
||||
}
|
||||
settling, stop := context.WithTimeout(b.ctx, requestWithin)
|
||||
defer stop()
|
||||
settle:
|
||||
for {
|
||||
select {
|
||||
case <-settling.Done():
|
||||
w.Stop()
|
||||
if b.ctx.Err() != nil {
|
||||
return b.ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("the watch of %s handed nothing in %s: a watch the bus refuses is answered by none", bucketOf(state), requestWithin)
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
w.Stop()
|
||||
return fmt.Errorf("the watch of %s ended before it held what stands", bucketOf(state))
|
||||
}
|
||||
if e == nil {
|
||||
break settle
|
||||
}
|
||||
put(e)
|
||||
}
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.settled[state] = true
|
||||
b.mu.Unlock()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-b.ctx.Done():
|
||||
w.Stop()
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
lost(errors.New("the server ended it"))
|
||||
return
|
||||
}
|
||||
if e != nil {
|
||||
put(e)
|
||||
var value []byte
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
changed(e.Key(), value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownStore is the part's own state on its process connection: a state it did not announce, or one the bus refuses,
|
||||
// fails the call, named.
|
||||
type ownStore struct{ c *process.Conn }
|
||||
|
||||
func (s ownStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.c.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
return s.c.State(state).Put(key, json.RawMessage(value))
|
||||
}
|
||||
|
||||
// emitting is the process connection as the part's emitter: its events, with no options.
|
||||
type emitting struct{ c *process.Conn }
|
||||
|
||||
func (e emitting) Emit(event string, body any) error { return e.c.Emit(event, body) }
|
||||
@@ -0,0 +1,128 @@
|
||||
// Command mesh-grants is the grants part (novox/hq ADR 0547 clauses 5 to 8, issue 1945): a process on its own bus
|
||||
// credential that watches placement, the assigner's wants, the indexer's manifests, the builder's builds, mesh-config's
|
||||
// resolved values and the node agents' haves, and on every change composes each assignment's membership, the bus's user
|
||||
// list and each manifest's buckets, logs and streams with the bus composer the controller uses (mesh-sdk buscompose).
|
||||
//
|
||||
// **Memberships are published for real** (the operator, 2026-10-25 22:31): since release 6 nobody publishes them, so a
|
||||
// module placed since cannot serve its seat or reach a new state. Each is published on the assignments stream only when
|
||||
// it differs from the one last published there. **The user list, the assertions and the grants stay in shadow** until
|
||||
// the operator's take-over, which this milestone always refuses: they are composed, compared with what the bus loaded
|
||||
// and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect.
|
||||
//
|
||||
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
|
||||
// event.
|
||||
//
|
||||
// **No seat in this milestone.** ADR 0547 clause 5 gives the part the seat `mesh-grants`; a `mesh-*` seat must be in
|
||||
// mesh-sdk's own seat list, which the running controller's composer does not know, so the part serves module tools
|
||||
// (grants_*) and claims no seat until take-over. The composer grants the membership publish by the module's name,
|
||||
// mesh-grants.
|
||||
//
|
||||
// Started by the tool runner, with no credential of its own, it serves those tools (a process never serves, ADR 0436).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||
if err := serveTools(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-grants:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-grants:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// keptStates is what the part keeps, each readable (ADR 0495 §2-§3): the view and a session read them.
|
||||
func keptStates() []announce.State {
|
||||
var out []announce.State
|
||||
for _, s := range grants.States {
|
||||
out = append(out, announce.State{Name: s, Readable: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// run is the process: it returns when ctx ends, or when it has no credential or cannot dial the bus at all. A bucket or
|
||||
// read the bus refuses never ends it.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-grants: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
|
||||
}
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := &grants.Follower{Logf: logger.Printf}
|
||||
conn, err := process.Connect(account, nats.Name(account.Node+"."+account.Module), nats.RetryOnFailedConnect(true),
|
||||
nats.ReconnectHandler(func(*nats.Conn) { f.Wake() }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.Keeps(keptStates()...); err != nil {
|
||||
logger.Printf("what the part keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
// The reads of other parts' state, on a connection of the same account: a read the bus refuses is said and composed
|
||||
// past (busReader), never the end of the process.
|
||||
opts := []nats.Option{
|
||||
nats.Name(account.Node + "." + account.Module + ".reads"),
|
||||
nats.UserInfo(account.User, account.Password),
|
||||
nats.CustomInboxPrefix(account.InboxPrefix()),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
nats.ReconnectHandler(func(*nats.Conn) { f.Wake() }),
|
||||
}
|
||||
if strings.TrimSpace(account.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
|
||||
}
|
||||
nc, err := nats.Connect(account.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bus := newBusReader(ctx, js)
|
||||
f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn},
|
||||
Publish: memberships{ctx: ctx, nc: nc, js: js}}
|
||||
f.Watch = bus
|
||||
f.Asked = func(wake func()) error {
|
||||
return conn.State(grants.StateUserList).Watch(grants.KeyAsked, func(c stdio.StateChange) error {
|
||||
if !c.Current {
|
||||
wake()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
logger.Printf("started on the account %s: publishing memberships; the user list, assertions and grants in shadow", account.User)
|
||||
f.Run(ctx)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// memberships publishes each membership on the assignments stream, last per subject, over the one last read there
|
||||
// (ExpectLastSequencePerSubject, so a membership another writer published meanwhile is not overwritten, issues 930 and
|
||||
// 934), and reads the last one by a direct get of its subject. The one place the part publishes on the bus.
|
||||
type memberships struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
func (m memberships) LastMembership(node, module string) ([]byte, uint64, bool, error) {
|
||||
ctx, cancel := context.WithTimeout(m.ctx, requestWithin)
|
||||
defer cancel()
|
||||
msg, err := m.nc.RequestWithContext(ctx, membership.DirectGetSubject(node, module), nil)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = fmt.Errorf("no answer in %s: a read the bus refuses is answered by none (%w)", requestWithin, err)
|
||||
}
|
||||
return nil, 0, false, err
|
||||
}
|
||||
switch status := msg.Header.Get("Status"); status {
|
||||
case "":
|
||||
case "404":
|
||||
return nil, 0, false, nil
|
||||
default:
|
||||
return nil, 0, false, fmt.Errorf("the assignments stream answered %s %s", status, msg.Header.Get("Description"))
|
||||
}
|
||||
seq, err := strconv.ParseUint(msg.Header.Get("Nats-Sequence"), 10, 64)
|
||||
if err != nil {
|
||||
return nil, 0, false, fmt.Errorf("the membership of %s on %s carries no sequence", module, node)
|
||||
}
|
||||
return msg.Data, seq, true, nil
|
||||
}
|
||||
|
||||
func (m memberships) PublishMembership(node, module string, body []byte, after uint64) error {
|
||||
ctx, cancel := context.WithTimeout(m.ctx, requestWithin)
|
||||
defer cancel()
|
||||
if _, err := m.js.Publish(ctx, membership.Subject(node, module), body, jetstream.WithExpectLastSequencePerSubject(after)); err != nil {
|
||||
return fmt.Errorf("the membership of %s on %s after sequence %d: %w", module, node, after, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
// The part's tools, served through the tool runner from its bundle (ADR 0436: a process never serves). They read and
|
||||
// write through the runtime as the module: its own states, the reads its manifest names, the bus module's read tools.
|
||||
|
||||
// runtimeReader reads another part's state through the runtime, `<module>.<state>`.
|
||||
type runtimeReader struct{}
|
||||
|
||||
func (runtimeReader) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
func (runtimeReader) Get(state, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
// runtimeStore is the part's own state through the runtime.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
return stdio.State(state).Put(key, json.RawMessage(value))
|
||||
}
|
||||
|
||||
// runtimeBus calls tools and says events through the runtime.
|
||||
type runtimeBus struct{}
|
||||
|
||||
func (runtimeBus) Call(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }
|
||||
func (runtimeBus) Emit(event string, body any) error { return stdio.Emit(event, body) }
|
||||
|
||||
func schema(properties map[string]string, required ...string) map[string]any {
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
props[name] = map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
out["required"] = required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// toolsOf are the part's tools over a part (grants.Tools names them).
|
||||
func toolsOf(p *grants.Part) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: grants.ToolGrants, Description: "What the grants part holds of grants (novox/hq ADR 0547 clause 6): in this milestone it " +
|
||||
"writes none (grants.<module>.<fp> from the third milestone, clause 5), and shows beside it the grants the controller " +
|
||||
"wrote (mesh-controller_grants), all or one module's with the accounts each names. Read-only.",
|
||||
Input: schema(map[string]string{"module": "one module"}),
|
||||
Run: func(a map[string]any) (any, error) { return p.Grants(grants.Answer(a, "module")) }},
|
||||
{Name: grants.ToolMembership, Description: "One module's membership on one node (ADR 0547 clause 6): as composed now from the " +
|
||||
"bus's records with the bus composer, and what the part's process published of it at its last pass (published, " +
|
||||
"unchanged, or why not: a shadow difference names what publishing would lose), with what the composition cannot " +
|
||||
"read. Read-only.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module"}, "node", "module"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.MembershipOf(grants.Answer(a, "node"), grants.Answer(a, "module"))
|
||||
}},
|
||||
{Name: grants.ToolUserList, Description: "The bus's user list in shadow (ADR 0547 clauses 6 and 8): the list the grants part " +
|
||||
"composed (digest, revision, users), what the bus node says it loaded (source, revision, digest, users, error), and how " +
|
||||
"they differ: users added, users dropped (and of those, gone from placement), each user's grant differences, the " +
|
||||
"assignments left out, every input the bus does not carry, named; and per user whether the vault holds its hash " +
|
||||
"(mesh-vault.bus-accounts, ADR 0547 clause 9) and how its line would differ from the loaded one (same, adds, left-out " +
|
||||
"for want of a credential, loses-login), never a hash. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return p.UserList() }},
|
||||
{Name: grants.ToolComposeNow, Description: "Compose again now (ADR 0547 clause 6), as the part does on every change it hears: asks " +
|
||||
"the part's process (which alone holds the grant to publish a membership) to compose the memberships, the user list and " +
|
||||
"the assertions, publish each membership that differs from the one last published, and keep what changed in its own " +
|
||||
"state (one revision when an input changed, none when not), saying user-list-changed and membership-changed. Answers " +
|
||||
"the ask and the last pass; grants_user_list shows the pass that answers it. A membership is published only when it is " +
|
||||
"the first or only adds to the last one; any loss stays a shadow difference, named. The user list, assertions and " +
|
||||
"grants stay in shadow. It cannot ask the controller to write a module's grant again: the controller serves no such " +
|
||||
"verb, and the answer says so.",
|
||||
Run: func(a map[string]any) (any, error) { return p.AskCompose(stdio.CallerOf(a)) }},
|
||||
{Name: grants.ToolAssert, Description: "One module's buckets, logs and streams (ADR 0547 clause 6), composed from its manifest at " +
|
||||
"the indexer's trunk with their limits, beside what the bus holds (through the bus module's nats_streams). In shadow it " +
|
||||
"asserts nothing, and says so.",
|
||||
Input: schema(map[string]string{"module": "the module"}, "module"),
|
||||
Run: func(a map[string]any) (any, error) { return p.AssertOne(grants.Answer(a, "module")) }},
|
||||
{Name: grants.ToolTakeOver, Description: "The operator's switch to the grants part's list (ADR 0547 clause 8), refused without the " +
|
||||
"operator's credential (a person's account), refused while any user of the shadow list differs from the loaded list (but " +
|
||||
"those placement no longer names), naming them; in this milestone always refused, since the enrolment users and the " +
|
||||
"persons cannot be composed. Every ask is recorded in the part's state takeover.",
|
||||
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return nil, p.TakeOver(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.ToolHandBack, Description: "The operator's switch back to the controller's list (ADR 0547 clause 8), refused without " +
|
||||
"the operator's credential; in shadow there is nothing to hand back, and it says so. Every ask is recorded.",
|
||||
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return nil, p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops.
|
||||
func serveTools() error {
|
||||
p := &grants.Part{Read: runtimeReader{}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}}
|
||||
return stdio.Serve("", toolsOf(p))
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
module mesh-grants
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025215021-b7b5c6ef27a0
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025215021-b7b5c6ef27a0 h1:8dKGrZzPthx/jZlV9Bjm2lWGNORUTETmq2vxKN+MRV0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025215021-b7b5c6ef27a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,811 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
// fakeBus is the bus as the part sees it: other parts' states to read (refusing those named), the bus module's read
|
||||
// tools, the part's own states and its events. It records every write, call and event, so a row can prove nothing was
|
||||
// published or asserted.
|
||||
type fakeBus struct {
|
||||
mu sync.Mutex
|
||||
states map[string]map[string][]byte
|
||||
refused map[string]bool
|
||||
loaded []grants.User
|
||||
streams []map[string]any
|
||||
own map[string]map[string][]byte
|
||||
revs map[string]uint64
|
||||
refuseTo map[string]bool
|
||||
calls []string
|
||||
events []string
|
||||
writes []string
|
||||
// The assignments stream: each subject's last membership and sequence, every publish, and whether it is refused.
|
||||
stream map[string][]byte
|
||||
seqs map[string]uint64
|
||||
seq uint64
|
||||
publishes []string
|
||||
refusePub bool
|
||||
// hold, when set, keeps every nats_users call waiting until it is closed: a pass waiting on the bus.
|
||||
hold chan struct{}
|
||||
}
|
||||
|
||||
func (b *fakeBus) LastMembership(node, module string) ([]byte, uint64, bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
subject := "mesh.assignment." + node + "." + module
|
||||
v, ok := b.stream[subject]
|
||||
return v, b.seqs[subject], ok, nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) PublishMembership(node, module string, body []byte, after uint64) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
subject := "mesh.assignment." + node + "." + module
|
||||
if b.refusePub {
|
||||
return errors.New("Permissions Violation for Publish to " + subject)
|
||||
}
|
||||
if b.seqs[subject] != after {
|
||||
return fmt.Errorf("wrong last sequence for %s: %d", subject, b.seqs[subject])
|
||||
}
|
||||
if b.stream == nil {
|
||||
b.stream, b.seqs = map[string][]byte{}, map[string]uint64{}
|
||||
}
|
||||
b.seq++
|
||||
b.stream[subject], b.seqs[subject] = body, b.seq
|
||||
b.publishes = append(b.publishes, subject)
|
||||
return nil
|
||||
}
|
||||
|
||||
func newFakeBus() *fakeBus {
|
||||
return &fakeBus{states: map[string]map[string][]byte{}, refused: map[string]bool{}, own: map[string]map[string][]byte{},
|
||||
revs: map[string]uint64{}, refuseTo: map[string]bool{}}
|
||||
}
|
||||
|
||||
func (b *fakeBus) put(state, key string, v any) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if b.states[state] == nil {
|
||||
b.states[state] = map[string][]byte{}
|
||||
}
|
||||
b.states[state][key] = raw
|
||||
}
|
||||
|
||||
func (b *fakeBus) Keys(state string) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.refused[state] {
|
||||
return nil, errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_" + state)
|
||||
}
|
||||
var out []string
|
||||
for k := range b.states[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.refused[state] {
|
||||
return nil, false, errors.New("refused")
|
||||
}
|
||||
v, ok := b.states[state][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
b.mu.Lock()
|
||||
hold := b.hold
|
||||
b.mu.Unlock()
|
||||
if hold != nil && key == grants.CallUsers {
|
||||
<-hold
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.calls = append(b.calls, key)
|
||||
args, _ := body.(map[string]any)
|
||||
switch key {
|
||||
case grants.CallUsers:
|
||||
part, _ := args["user"].(string)
|
||||
if part == "" {
|
||||
var names []map[string]any
|
||||
for _, u := range b.loaded {
|
||||
names = append(names, map[string]any{"user": u.User, "publish_grants": len(u.Publish)})
|
||||
}
|
||||
return json.Marshal(map[string]any{"users": names})
|
||||
}
|
||||
var out []grants.User
|
||||
for _, u := range b.loaded {
|
||||
if strings.Contains(u.User, part) {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
return json.Marshal(map[string]any{"users": out})
|
||||
case grants.CallStreams:
|
||||
return json.Marshal(map[string]any{"detail": b.streams})
|
||||
}
|
||||
return nil, fmt.Errorf("no tool %s", key)
|
||||
}
|
||||
|
||||
func (b *fakeBus) ownGet(state, key string) ([]byte, uint64, bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
v, ok := b.own[state][key]
|
||||
return v, b.revs[state+"/"+key], ok, nil
|
||||
}
|
||||
|
||||
type ownStore struct{ b *fakeBus }
|
||||
|
||||
func (s ownStore) Get(state, key string) ([]byte, uint64, bool, error) { return s.b.ownGet(state, key) }
|
||||
func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
b := s.b
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.refuseTo[state] {
|
||||
return 0, errors.New("Permissions Violation for Publish to $KV.mesh-grants_" + state)
|
||||
}
|
||||
b.writes = append(b.writes, state)
|
||||
if b.own[state] == nil {
|
||||
b.own[state] = map[string][]byte{}
|
||||
}
|
||||
b.own[state][key] = value
|
||||
b.revs[state+"/"+key]++
|
||||
return b.revs[state+"/"+key], nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Emit(event string, _ any) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.events = append(b.events, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) count(key string) int {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range b.calls {
|
||||
if c == key {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// manifestOf is a fixture manifest of one module, in the indexer's two buckets.
|
||||
func (b *fakeBus) manifestOf(module, raw string) {
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw})
|
||||
b.put(grants.ReadModules, module, delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest, Repository: "novox/x", Commit: "c"})
|
||||
}
|
||||
|
||||
func (b *fakeBus) assign(node, module string) {
|
||||
b.put(grants.ReadAssignments, node+"."+module, map[string]any{"node": node, "module": module, "assigned": "2026-10-25T20:00:00Z"})
|
||||
}
|
||||
|
||||
const (
|
||||
natsManifest = `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"}}`
|
||||
xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` +
|
||||
`"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history"}]}`
|
||||
yManifest = `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"reads":["mod-x.kept"]}`
|
||||
)
|
||||
|
||||
// fixture is a mesh of two machines: a runs the bus and mod-x, b runs mod-y; the bus node's agent loaded a list.
|
||||
func fixture() *fakeBus {
|
||||
b := newFakeBus()
|
||||
b.manifestOf("nats", natsManifest)
|
||||
b.manifestOf("mod-x", xManifest)
|
||||
b.manifestOf("mod-y", yManifest)
|
||||
b.assign("a", "nats")
|
||||
b.assign("a", "mod-x")
|
||||
b.assign("b", "mod-y")
|
||||
b.put(grants.ReadAssignments, "b.gone", map[string]any{"node": "b", "module": "gone", "assigned": "2026-10-01T00:00:00Z", "ended": "2026-10-02T00:00:00Z"})
|
||||
b.put(grants.ReadHave, "a.node", map[string]any{"v": 1})
|
||||
b.put(grants.ReadHave, "b.node", map[string]any{"v": 1})
|
||||
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 6}})
|
||||
b.put(grants.ReadControllerGrant, "mod-x.0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123", delivery.ControllerGrant{V: delivery.Version, Accounts: []string{"a.mod-x"}})
|
||||
return b
|
||||
}
|
||||
|
||||
func partOn(b *fakeBus) *grants.Part {
|
||||
return &grants.Part{Read: b, Call: b, Store: ownStore{b}, Emit: b, Publish: b, Now: func() time.Time { return time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC) }}
|
||||
}
|
||||
|
||||
// shadowOf is the list the part composes from b's records now.
|
||||
func shadowOf(t *testing.T, b *fakeBus) []grants.User {
|
||||
t.Helper()
|
||||
recs, _ := grants.Records(grants.ReadInputs(b, &grants.Manifests{}))
|
||||
users, err := grants.ComposeUsers(recs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
func names(us []grants.User) []string {
|
||||
var out []string
|
||||
for _, u := range us {
|
||||
out = append(out, u.User)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func gapNamed(gs []grants.Gap, input string) bool {
|
||||
return slices.ContainsFunc(gs, func(g grants.Gap) bool { return strings.Contains(g.Input, input) })
|
||||
}
|
||||
|
||||
// TestGrants is the grants part's one table of checks (ADR 0547 clauses 5 to 8 in shadow, issue 1945): its manifest,
|
||||
// the shadow list composed from bus values, the compare, each tool, that nothing is published or asserted, and that a
|
||||
// refused read or write never stops it.
|
||||
func TestGrants(t *testing.T) {
|
||||
t.Run("manifest", func(t *testing.T) {
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err)
|
||||
}
|
||||
if m.Module != "mesh-grants" || len(m.Claims) != 0 {
|
||||
t.Errorf("module %q claims %v: the part claims no seat until mesh-sdk's seat list names mesh-grants", m.Module, m.Claims)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
what string
|
||||
got, want []string
|
||||
}{
|
||||
{"reads", m.Reads, grants.Reads},
|
||||
{"invokes", m.Invokes, grants.Invokes},
|
||||
{"emits", m.Emits, grants.Emits},
|
||||
{"tools", m.Tools, grants.Tools},
|
||||
} {
|
||||
if !slices.Equal(slices.Sorted(slices.Values(row.got)), slices.Sorted(slices.Values(row.want))) {
|
||||
t.Errorf("%s = %v, the program's are %v", row.what, row.got, row.want)
|
||||
}
|
||||
}
|
||||
var states []string
|
||||
for _, s := range m.State {
|
||||
states = append(states, s.Name)
|
||||
}
|
||||
if !slices.Equal(slices.Sorted(slices.Values(states)), grants.States) {
|
||||
t.Errorf("state = %v, want %v", states, grants.States)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || m.SecretsOwner != "mesh-grants" {
|
||||
t.Errorf("own-secrets %v owner %q: a credential of its own, broker", m.OwnSecrets, m.SecretsOwner)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no timer, no create, a publish only of memberships", func(t *testing.T) {
|
||||
forbidden := []string{"time.Tick", "time.NewTicker", "time.Sleep", "time.After", "time.NewTimer", "time.AfterFunc",
|
||||
"PublishMsg", "CreateStream", "CreateKeyValue", "CreateOrUpdate", "UpdateStream", "UpdateKeyValue",
|
||||
"CreateObjectStore", "EnsureStream", "RaiseBuckets", "RaiseLogs", ".Create("}
|
||||
// The one publish: a membership, on its own subject, in publish.go.
|
||||
publishOnly := filepath.Join("cmd", "mesh-grants", "publish.go")
|
||||
files := 0
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
src, err := os.ReadFile(path)
|
||||
if path != publishOnly && (strings.Contains(string(src), ".Publish(") || strings.Contains(string(src), "mesh.assignment")) {
|
||||
t.Errorf("%s publishes: only publish.go publishes, and only memberships", path)
|
||||
}
|
||||
if path == publishOnly && strings.Count(string(src), ".Publish(") != 1 {
|
||||
t.Errorf("publish.go publishes other than one membership")
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the part holds no timer and creates or asserts nothing", path, f)
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if files < 8 {
|
||||
t.Fatalf("read %d source files, want the program's", files)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("shadow list from bus values", func(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
bend func(*fakeBus)
|
||||
users []string
|
||||
gap string
|
||||
}{
|
||||
{"every record on the bus", func(*fakeBus) {},
|
||||
[]string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"},
|
||||
{"a manifest missing is a gap and its user is left out", func(b *fakeBus) { delete(b.states[grants.ReadModules], "mod-y") },
|
||||
[]string{"a.mod-x", "a.nats", "controller", "node.a", "node.b"}, "manifest of mod-y"},
|
||||
{"a want's built manifest is composed, not the trunk's", func(b *fakeBus) {
|
||||
built := strings.Replace(xManifest, `"history":3}]`, `"history":3},{"name":"newer"}]`, 1)
|
||||
digest := delivery.ManifestDigest([]byte(built))
|
||||
b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: "mod-x", JSON: built})
|
||||
fp := "src2:" + strings.Repeat("cd", 32)
|
||||
b.put(grants.ReadWants, "want.a.mod-x", delivery.Want{V: delivery.Version, FP: fp, Composed: "sha256:" + strings.Repeat("ef", 32)})
|
||||
key, _ := delivery.BuilderBuildKey("mod-x", fp)
|
||||
b.put(grants.ReadBuilds, key, map[string]any{"v": delivery.Version, "state": "done", "manifest": digest})
|
||||
}, []string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"},
|
||||
{"placement refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
|
||||
[]string{"controller", "node.a", "node.b"}, grants.ReadAssignments},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := fixture()
|
||||
row.bend(b)
|
||||
in := grants.ReadInputs(b, &grants.Manifests{})
|
||||
recs, left := grants.Records(in)
|
||||
users, err := grants.ComposeUsers(recs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := names(users); !slices.Equal(got, row.users) {
|
||||
t.Errorf("users = %v, want %v", got, row.users)
|
||||
}
|
||||
if !gapNamed(append(in.Gaps, left...), row.gap) {
|
||||
t.Errorf("gaps %v name no %q", in.Gaps, row.gap)
|
||||
}
|
||||
if strings.HasPrefix(row.name, "a want's") {
|
||||
x := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.mod-x" })]
|
||||
if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") {
|
||||
t.Errorf("a.mod-x is not granted the state its wanted build declares: %v", x.Publish)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
b := fixture()
|
||||
users := shadowOf(t, b)
|
||||
y := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "b.mod-y" })]
|
||||
if !slices.Contains(y.Publish, "$JS.API.DIRECT.GET.KV_mod-x_kept.>") {
|
||||
t.Errorf("b.mod-y reads mod-x.kept and is not granted its direct get: %v", y.Publish)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("credentials", func(t *testing.T) {
|
||||
shadow := []grants.User{{User: "a.held"}, {User: "a.new"}, {User: "a.bare"}}
|
||||
loaded := []grants.User{{User: "a.held"}, {User: "a.bare"}, {User: "enrol.x"}, {User: "person.op"}}
|
||||
hashes := map[string]grants.HeldHash{"a.held": {Hash: "$2a$10$h1", Generation: 2}, "a.new": {Hash: "$2a$10$h2", Generation: 1},
|
||||
"person.op": {Hash: "$2a$10$h3", Adopted: true}}
|
||||
c := grants.CredentialsOf(shadow, loaded, hashes)
|
||||
for _, row := range []struct {
|
||||
user, would string
|
||||
held, adopted bool
|
||||
}{
|
||||
{"a.bare", grants.WouldLeaveOut, false, false},
|
||||
{"a.held", grants.WouldSame, true, false},
|
||||
{"a.new", grants.WouldAdd, true, false},
|
||||
{"enrol.x", grants.WouldLoseLogin, false, false},
|
||||
{"person.op", grants.WouldNotComposed, true, true},
|
||||
} {
|
||||
i := slices.IndexFunc(c.Users, func(u grants.Credential) bool { return u.User == row.user })
|
||||
if i < 0 || c.Users[i].Would != row.would || c.Users[i].HashHeld != row.held || c.Users[i].Adopted != row.adopted {
|
||||
t.Errorf("%s: %+v, want %s held %v", row.user, c.Users, row.would, row.held)
|
||||
}
|
||||
}
|
||||
if c.HashesHeld != 3 || !slices.Equal(c.LeftOut, []string{"a.bare"}) || !slices.Equal(c.LosesLogin, []string{"enrol.x"}) ||
|
||||
!slices.Equal(c.Adds, []string{"a.new"}) {
|
||||
t.Errorf("credentials = %+v", c)
|
||||
}
|
||||
// Composed from the vault's state: a hash changes the list's state (written on the event), and no hash is ever
|
||||
// in what the part writes or says.
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
before := p.Compose()
|
||||
b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": "$2a$10$secrethashbytes", "generation": 1})
|
||||
after := p.Compose()
|
||||
cr := after.UserList.Credentials
|
||||
if !after.Written || after.Revision != before.Revision+1 || cr.HashesHeld != 1 || slices.Contains(cr.LeftOut, "a.mod-x") ||
|
||||
!slices.Contains(cr.LeftOut, "b.mod-y") {
|
||||
t.Errorf("a hash heard = %+v", after.UserList.Credentials)
|
||||
}
|
||||
err := p.TakeOver("person.operator", "hashes")
|
||||
if err == nil || !strings.Contains(err.Error(), "no hash for") || strings.Contains(err.Error(), "a.mod-x,") {
|
||||
t.Errorf("take-over = %v, want the users with no hash named", err)
|
||||
}
|
||||
for state, keys := range b.own {
|
||||
for k, v := range keys {
|
||||
if strings.Contains(string(v), "secrethashbytes") {
|
||||
t.Errorf("the hash is written in %s %s", state, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("compare", func(t *testing.T) {
|
||||
b := fixture()
|
||||
shadow := shadowOf(t, b)
|
||||
in := grants.ReadInputs(b, &grants.Manifests{})
|
||||
changed := slices.Clone(shadow)
|
||||
x := slices.IndexFunc(changed, func(u grants.User) bool { return u.User == "a.mod-x" })
|
||||
changed[x].Publish = append(slices.Clone(changed[x].Publish[1:]), "extra.subject")
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
loaded []grants.User
|
||||
added, dropped, gone []string
|
||||
differ []string
|
||||
equal bool
|
||||
}{
|
||||
{"equal", shadow, nil, nil, nil, nil, true},
|
||||
{"one user the bus lacks", shadow[1:], []string{shadow[0].User}, nil, nil, nil, false},
|
||||
{"one loaded user still placed", append(slices.Clone(shadow), grants.User{User: "a.mod-x.worker"}), nil, []string{"a.mod-x.worker"}, nil, nil, false},
|
||||
{"one loaded user placement no longer names", append(slices.Clone(shadow), grants.User{User: "b.gone"}), nil, []string{"b.gone"}, []string{"b.gone"}, nil, true},
|
||||
{"grants differ", changed, nil, nil, nil, []string{"a.mod-x"}, false},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
c := grants.Compare(shadow, row.loaded, grants.Placed(in))
|
||||
var differ []string
|
||||
for _, d := range c.Differ {
|
||||
differ = append(differ, d.User)
|
||||
}
|
||||
if !slices.Equal(c.Added, nonNil(row.added)) || !slices.Equal(c.Dropped, nonNil(row.dropped)) ||
|
||||
!slices.Equal(c.Gone, row.gone) || !slices.Equal(differ, row.differ) || c.Equal() != row.equal {
|
||||
t.Errorf("compare = %+v", c)
|
||||
}
|
||||
if row.name == "grants differ" && (len(c.Differ[0].PublishAdded) != 1 || c.Differ[0].PublishDropped[0] != "extra.subject") {
|
||||
t.Errorf("the grant difference is not named: %+v", c.Differ[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("tools", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
b.streams = []map[string]any{{"name": "KV_mod-x_kept", "subjects": []string{"$KV.mod-x_kept.>"}, "retention": "limits", "max_per_subject": 3}}
|
||||
p := partOn(b)
|
||||
|
||||
got, _ := p.UserList()
|
||||
if !strings.Contains(fmt.Sprint(got), "nothing composed yet") {
|
||||
t.Errorf("user-list before any composition = %v", got)
|
||||
}
|
||||
first := p.Compose()
|
||||
if !first.Written || first.Revision != 1 || !first.UserList.Compare.Equal() || first.UserList.Users != 6 {
|
||||
t.Fatalf("first composition = %+v", first)
|
||||
}
|
||||
if len(first.Published) != 3 || len(b.publishes) != 3 {
|
||||
t.Errorf("the first composition published %v, want the three assignments' memberships", first.Published)
|
||||
}
|
||||
// The second pass says each membership unchanged (published at the first); the third writes nothing at all.
|
||||
if again := p.Compose(); again.Written || again.Revision != 1 || len(again.Published) != 0 || again.AssertionsWritten != 0 {
|
||||
t.Errorf("a composition with no input changed wrote or published: %+v", again)
|
||||
}
|
||||
if third := p.Compose(); third.Written || third.MembershipsWritten != 0 || len(third.Published) != 0 || len(b.publishes) != 3 {
|
||||
t.Errorf("a membership equal to the one last published was published again, or a state written: %+v", third)
|
||||
}
|
||||
// mod-x placed on b too: b.mod-x is added and its first membership published; a.mod-x's would lose the module's
|
||||
// plain subject (mod-x now runs on two machines), a loss, so it stays a shadow difference.
|
||||
b.assign("b", "mod-x")
|
||||
moved := p.Compose()
|
||||
if !moved.Written || moved.Revision != 2 || !slices.Equal(moved.Published, []string{"b.mod-x"}) || len(moved.NotPublished) != 1 || !strings.HasPrefix(moved.NotPublished[0], "a.mod-x: "+grants.ShadowDifference) || moved.UserList.Compare.Equal() ||
|
||||
!slices.Equal(moved.UserList.Compare.Added, []string{"b.mod-x"}) {
|
||||
t.Errorf("a composition after placement moved = %+v", moved)
|
||||
}
|
||||
got, _ = p.UserList()
|
||||
ul, _ := got.(map[string]any)
|
||||
if st, ok := ul["written"].(grants.UserListState); !ok || st.Loaded == nil || st.Loaded.Digest != "sha256:aa" || st.BusNode != "a" ||
|
||||
ul["revision"] != uint64(2) || !gapNamed(st.Gaps, "people") {
|
||||
t.Errorf("user-list = %+v", got)
|
||||
}
|
||||
|
||||
a, err := p.AssertOne("mod-x")
|
||||
as, _ := a.(grants.Assertion)
|
||||
if err != nil || as.Asserted || as.Says != grants.ShadowAsserts || len(as.Items) != 3 {
|
||||
t.Fatalf("assert = %+v, %v", a, err)
|
||||
}
|
||||
for _, it := range as.Items {
|
||||
if it.Stream == "KV_mod-x_kept" && len(it.Differences) != 0 {
|
||||
t.Errorf("a bucket the bus holds as composed differs: %v", it.Differences)
|
||||
}
|
||||
if it.Stream == "THINGS" && !slices.Contains(it.Differences, "the bus holds no stream THINGS") {
|
||||
t.Errorf("a stream the bus lacks is not named: %+v", it)
|
||||
}
|
||||
}
|
||||
|
||||
ms, _ := p.MembershipOf("a", "mod-x")
|
||||
if mo, _ := ms.(map[string]any); !strings.Contains(fmt.Sprint(mo["published"]), "last_pass") || mo["composed"] == nil {
|
||||
t.Errorf("membership = %v", ms)
|
||||
}
|
||||
gr, _ := p.Grants("mod-x")
|
||||
if g, _ := gr.(map[string]any); g["writes"] != grants.WritesNoGrant || g["count"] != 1 {
|
||||
t.Errorf("grants = %v", gr)
|
||||
}
|
||||
|
||||
b.loaded = append(b.loaded, grants.User{User: "b.gone"})
|
||||
for _, row := range []struct {
|
||||
name, act, caller string
|
||||
says, never []string
|
||||
}{
|
||||
{"take-over without the operator's credential", "take-over", "shanks.node-tools", []string{"operator's alone"}, nil},
|
||||
{"hand-back without the operator's credential", "hand-back", "unproven", []string{"operator's alone"}, nil},
|
||||
{"take-over while a user differs names it", "take-over", "person.operator", []string{"users differ", "b.mod-x", grants.NotComposable}, []string{"b.gone"}},
|
||||
{"hand-back in shadow", "hand-back", "person.operator", []string{"nothing to hand back"}, nil},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
var err error
|
||||
if row.act == "take-over" {
|
||||
err = p.TakeOver(row.caller, "a row")
|
||||
} else {
|
||||
err = p.HandBack(row.caller, "a row")
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatalf("%s by %s was not refused", row.act, row.caller)
|
||||
}
|
||||
for _, s := range row.says {
|
||||
if !strings.Contains(err.Error(), s) {
|
||||
t.Errorf("refusal %q does not say %q", err, s)
|
||||
}
|
||||
}
|
||||
for _, s := range row.never {
|
||||
if strings.Contains(err.Error(), s) {
|
||||
t.Errorf("refusal %q names %q", err, s)
|
||||
}
|
||||
}
|
||||
raw, _, _, _ := b.ownGet(grants.StateTakeover, grants.KeySwitch)
|
||||
var s grants.Switch
|
||||
_ = json.Unmarshal(raw, &s)
|
||||
last := s.Acts[len(s.Acts)-1]
|
||||
if s.State != grants.Shadow || last.Act != row.act || last.By != row.caller || last.Refused == "" {
|
||||
t.Errorf("the act is not recorded: %+v", s)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("take-over with only a user placement no longer names is refused for want of records alone", func(t *testing.T) {
|
||||
b.states[grants.ReadAssignments] = nil
|
||||
b.assign("a", "nats")
|
||||
b.assign("a", "mod-x")
|
||||
b.assign("b", "mod-y")
|
||||
b.loaded = append(shadowOf(t, b), grants.User{User: "b.gone"})
|
||||
err := p.TakeOver("person.operator", "lists agree")
|
||||
if err == nil || strings.Contains(err.Error(), "differ") || !strings.Contains(err.Error(), grants.NotComposable) {
|
||||
t.Errorf("take-over = %v, want refused for the enrolment users and persons alone", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("only memberships published, nothing asserted", func(t *testing.T) {
|
||||
for _, s := range b.publishes {
|
||||
if !strings.HasPrefix(s, "mesh.assignment.") {
|
||||
t.Errorf("published on %s", s)
|
||||
}
|
||||
}
|
||||
for _, w := range b.writes {
|
||||
if !slices.Contains(grants.States, w) {
|
||||
t.Errorf("wrote %s, which is not the part's own state", w)
|
||||
}
|
||||
}
|
||||
for _, c := range b.calls {
|
||||
if c != grants.CallUsers && c != grants.CallStreams {
|
||||
t.Errorf("called %s: the part calls only the bus module's read tools", c)
|
||||
}
|
||||
}
|
||||
for _, e := range b.events {
|
||||
if !slices.Contains(grants.Emits, e) {
|
||||
t.Errorf("said %s", e)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("memberships: a first or an additive one published, a loss never", func(t *testing.T) {
|
||||
const key = "a.mod-x"
|
||||
composed := func(b *fakeBus) map[string]any {
|
||||
recs, _ := grants.Records(grants.ReadInputs(b, &grants.Manifests{}))
|
||||
raw, _ := json.Marshal(grants.Memberships(recs)[key].Membership)
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
return m
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
last func(map[string]any) map[string]any
|
||||
published bool
|
||||
names string
|
||||
}{
|
||||
{"a first membership is published", nil, true, ""},
|
||||
{"an additive change is published, the private network carried", func(m map[string]any) map[string]any {
|
||||
delete(m, "state")
|
||||
m["mesh"] = []any{"10.0.0.1"}
|
||||
return m
|
||||
}, true, ""},
|
||||
{"a change that drops verified-sender is not published, and named", func(m map[string]any) map[string]any {
|
||||
m["seat-traffic"] = map[string]any{"kinds": []any{map[string]any{"seat": "channel", "kind": "telegram", "capabilities": []any{"verified-sender"}}}}
|
||||
return m
|
||||
}, false, "verified-sender"},
|
||||
{"a change that drops any other field is not published", func(m map[string]any) map[string]any {
|
||||
m["serves"] = append(m["serves"].([]any), map[string]any{"subject": "mesh.mod.mod-x.tool.gone"})
|
||||
return m
|
||||
}, false, "tool.gone"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
if row.last != nil {
|
||||
raw, _ := json.Marshal(row.last(composed(b)))
|
||||
b.stream = map[string][]byte{"mesh.assignment." + key: raw}
|
||||
b.seqs = map[string]uint64{"mesh.assignment." + key: 4}
|
||||
}
|
||||
res := partOn(b).Compose()
|
||||
if got := slices.Contains(res.Published, key); got != row.published {
|
||||
t.Fatalf("published %v, not published %v", res.Published, res.NotPublished)
|
||||
}
|
||||
if row.published {
|
||||
if row.last != nil && !strings.Contains(string(b.stream["mesh.assignment."+key]), "10.0.0.1") {
|
||||
t.Errorf("the private network of the last membership is not carried: %s", b.stream["mesh.assignment."+key])
|
||||
}
|
||||
return
|
||||
}
|
||||
said := strings.Join(res.NotPublished, "\n")
|
||||
if !strings.Contains(said, grants.ShadowDifference) || !strings.Contains(said, row.names) {
|
||||
t.Errorf("the loss is not named: %s", said)
|
||||
}
|
||||
ms, _ := partOn(b).MembershipOf("a", "mod-x")
|
||||
if !strings.Contains(fmt.Sprint(ms), row.names) {
|
||||
t.Errorf("grants_membership does not name what would be lost: %v", ms)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("a refused publish: no exit, said, published at the next event", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
b.refusePub = true
|
||||
p := partOn(b)
|
||||
refused := p.Compose()
|
||||
if len(refused.Published) != 0 || len(refused.NotPublished) != 3 || !strings.Contains(refused.NotPublished[0], "Permissions Violation") {
|
||||
t.Errorf("a refused publish = %+v: said, and nothing published", refused)
|
||||
}
|
||||
b.refusePub = false
|
||||
if next := p.Compose(); !slices.Equal(next.Published, []string{"a.mod-x", "a.nats", "b.mod-y"}) {
|
||||
t.Errorf("the next event published %v", next.Published)
|
||||
}
|
||||
})
|
||||
t.Run("a tool answers while a pass waits on the bus", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
b.hold = make(chan struct{})
|
||||
p := partOn(b)
|
||||
done := make(chan struct{})
|
||||
go func() { p.Compose(); close(done) }()
|
||||
answered := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = p.UserList()
|
||||
_, _ = p.MembershipOf("a", "mod-x")
|
||||
_, _ = p.Grants("")
|
||||
_ = p.Last()
|
||||
close(answered)
|
||||
}()
|
||||
select {
|
||||
case <-answered:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Error("a tool waited on the pass's round trip")
|
||||
}
|
||||
close(b.hold)
|
||||
<-done
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("a refused read or write never stops it", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
b.refuseTo[grants.StateMemberships] = true
|
||||
res := partOn(b).Compose()
|
||||
if !res.Written || len(res.Errors) == 0 || !strings.Contains(strings.Join(res.Errors, " "), "memberships") {
|
||||
t.Errorf("a refused write of memberships = %+v: the user list is written all the same, the refusal said", res)
|
||||
}
|
||||
w := &fakeWatcher{refuse: map[string]int{grants.ReadModules: 1}, opened: make(chan string, 16)}
|
||||
var logged []string
|
||||
var lmu sync.Mutex
|
||||
f := &grants.Follower{Part: partOn(b), Watch: w, Logf: func(format string, args ...any) {
|
||||
lmu.Lock()
|
||||
logged = append(logged, fmt.Sprintf(format, args...))
|
||||
lmu.Unlock()
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { f.Run(ctx); close(done) }()
|
||||
waitOpened(t, w, grants.ReadAssignments)
|
||||
if r := f.Refused(); r[grants.ReadModules] == "" {
|
||||
waitFor(t, func() bool { return f.Refused()[grants.ReadModules] != "" })
|
||||
}
|
||||
w.change(grants.ReadAssignments, "a.mod-x")
|
||||
waitOpened(t, w, grants.ReadModules)
|
||||
waitFor(t, func() bool { return len(f.Refused()) == 0 })
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("the follower stopped")
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
lmu.Lock()
|
||||
defer lmu.Unlock()
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "cannot be watched yet") {
|
||||
t.Errorf("the refusal was not said: %v", logged)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func nonNil(s []string) []string {
|
||||
if s == nil {
|
||||
return []string{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fakeWatcher refuses a state's first watches as told, and hands changes to the watches it opened.
|
||||
type fakeWatcher struct {
|
||||
mu sync.Mutex
|
||||
refuse map[string]int
|
||||
changes map[string]func(string, []byte)
|
||||
opened chan string
|
||||
}
|
||||
|
||||
func (w *fakeWatcher) Watch(state string, changed func(string, []byte), _ func(error)) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.refuse[state] > 0 {
|
||||
w.refuse[state]--
|
||||
return errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_" + state)
|
||||
}
|
||||
if w.changes == nil {
|
||||
w.changes = map[string]func(string, []byte){}
|
||||
}
|
||||
w.changes[state] = changed
|
||||
w.opened <- state
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *fakeWatcher) change(state, key string) {
|
||||
w.mu.Lock()
|
||||
h := w.changes[state]
|
||||
w.mu.Unlock()
|
||||
h(key, []byte(`{}`))
|
||||
}
|
||||
|
||||
func waitOpened(t *testing.T, w *fakeWatcher, state string) {
|
||||
t.Helper()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case s := <-w.opened:
|
||||
if s == state {
|
||||
return
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("%s was never watched", state)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
for end := time.Now().Add(10 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("never came")
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Item is one bucket, log or stream a manifest declares, as composed, beside what the bus holds of it.
|
||||
type Item struct {
|
||||
Kind string `json:"kind"`
|
||||
Stream string `json:"stream"`
|
||||
// Composed is what an assertion would ask for; Held what the bus holds, absent when it holds no such stream.
|
||||
Composed Limits `json:"composed"`
|
||||
Held *Limits `json:"held,omitempty"`
|
||||
// Differences says how they differ, in words; empty when they agree in what can be read from here.
|
||||
Differences []string `json:"differences,omitempty"`
|
||||
}
|
||||
|
||||
// Limits are the facts of a stream an assertion sets and the bus module's nats_streams reads back. Its size cap is not
|
||||
// read back by that tool, so it is composed and not compared.
|
||||
type Limits struct {
|
||||
Subjects []string `json:"subjects"`
|
||||
Retention string `json:"retention"`
|
||||
MaxAgeHours int `json:"max_age_hours,omitempty"`
|
||||
MaxPerSubject int `json:"max_per_subject,omitempty"`
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// Assertion is one module's declared buckets, logs and streams, composed and compared. Asserted is always false in
|
||||
// shadow: nothing is created or changed.
|
||||
type Assertion struct {
|
||||
Module string `json:"module"`
|
||||
Manifest string `json:"manifest"`
|
||||
Items []Item `json:"items"`
|
||||
Asserted bool `json:"asserted"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// ShadowAsserts is what every assertion says in this milestone.
|
||||
const ShadowAsserts = "in shadow: composed and compared, nothing asserted (ADR 0547 clause 5 asserts from take-over)"
|
||||
|
||||
// Composed is a module's buckets, logs and streams as an assertion would ask for them. Pure.
|
||||
func Composed(m catalogue.Manifest) []Item {
|
||||
var out []Item
|
||||
for _, b := range bucketsOf(m) {
|
||||
per := b.History
|
||||
if per < 1 {
|
||||
per = 1
|
||||
}
|
||||
out = append(out, Item{Kind: "bucket", Stream: "KV_" + b.Bucket(), Composed: Limits{
|
||||
Subjects: []string{"$KV." + b.Bucket() + ".>"}, Retention: string(buscompose.RetentionLimits),
|
||||
MaxAgeHours: b.TTLSeconds / 3600, MaxPerSubject: per}})
|
||||
}
|
||||
for _, l := range logsOf(m) {
|
||||
c := l.Config()
|
||||
out = append(out, Item{Kind: "log", Stream: l.Stream(), Composed: Limits{Subjects: set(c.Subjects),
|
||||
Retention: string(buscompose.RetentionLimits), MaxAgeHours: int(c.MaxAge.Hours()), MaxBytes: c.MaxBytes}})
|
||||
}
|
||||
for _, s := range streamsOf(m) {
|
||||
st := s.Stream()
|
||||
out = append(out, Item{Kind: "stream", Stream: st.Name, Composed: Limits{Subjects: set(st.Subjects),
|
||||
Retention: string(st.Retention), MaxAgeHours: st.MaxAge / 3600, MaxBytes: st.MaxBytes}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HeldStreams reads every stream the bus holds through its module's nats_streams, by name.
|
||||
func HeldStreams(c Caller) (map[string]Limits, error) {
|
||||
raw, err := c.Call(CallStreams, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", CallStreams, err)
|
||||
}
|
||||
var got struct {
|
||||
Detail []struct {
|
||||
Name string `json:"name"`
|
||||
Subjects []string `json:"subjects"`
|
||||
Retention string `json:"retention"`
|
||||
MaxAgeHours int `json:"max_age_hours"`
|
||||
MaxPerSubject int `json:"max_per_subject"`
|
||||
} `json:"detail"`
|
||||
}
|
||||
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
|
||||
return nil, fmt.Errorf("%s answered no streams: %w", CallStreams, err)
|
||||
}
|
||||
out := map[string]Limits{}
|
||||
for _, s := range got.Detail {
|
||||
out[s.Name] = Limits{Subjects: set(s.Subjects), Retention: s.Retention, MaxAgeHours: s.MaxAgeHours, MaxPerSubject: s.MaxPerSubject}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Assert composes a module's items and compares each with what the bus holds; held nil is not read. It asserts nothing.
|
||||
func Assert(module, digest string, m catalogue.Manifest, held map[string]Limits, heldErr error) Assertion {
|
||||
a := Assertion{Module: module, Manifest: digest, Items: Composed(m), Says: ShadowAsserts}
|
||||
for i := range a.Items {
|
||||
it := &a.Items[i]
|
||||
if heldErr != nil {
|
||||
it.Differences = []string{"what the bus holds was not read: " + heldErr.Error()}
|
||||
continue
|
||||
}
|
||||
h, ok := held[it.Stream]
|
||||
if !ok {
|
||||
it.Differences = []string{"the bus holds no stream " + it.Stream}
|
||||
continue
|
||||
}
|
||||
it.Held = &h
|
||||
if !slices.Equal(h.Subjects, it.Composed.Subjects) {
|
||||
it.Differences = append(it.Differences, fmt.Sprintf("subjects %v held, %v composed", h.Subjects, it.Composed.Subjects))
|
||||
}
|
||||
if h.Retention != it.Composed.Retention {
|
||||
it.Differences = append(it.Differences, fmt.Sprintf("retention %s held, %s composed", h.Retention, it.Composed.Retention))
|
||||
}
|
||||
if h.MaxAgeHours != it.Composed.MaxAgeHours {
|
||||
it.Differences = append(it.Differences, fmt.Sprintf("age %d h held, %d h composed", h.MaxAgeHours, it.Composed.MaxAgeHours))
|
||||
}
|
||||
if it.Kind == "bucket" && max(h.MaxPerSubject, 1) != it.Composed.MaxPerSubject {
|
||||
it.Differences = append(it.Differences, fmt.Sprintf("history %d held, %d composed", h.MaxPerSubject, it.Composed.MaxPerSubject))
|
||||
}
|
||||
}
|
||||
sort.SliceStable(a.Items, func(i, j int) bool { return a.Items[i].Stream < a.Items[j].Stream })
|
||||
return a
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Watcher watches another part's state: changed hears each change after what stands, ended hears the watch end. A watch
|
||||
// refused (a read not granted yet, a bucket not there) is an error, and is tried again at the next wake.
|
||||
type Watcher interface {
|
||||
Watch(state string, changed func(key string, value []byte), ended func(error)) error
|
||||
}
|
||||
|
||||
// Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
|
||||
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
|
||||
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
|
||||
var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts}
|
||||
|
||||
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
|
||||
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
|
||||
type Follower struct {
|
||||
Part *Part
|
||||
Watch Watcher
|
||||
Logf func(string, ...any)
|
||||
// Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none.
|
||||
Asked func(wake func()) error
|
||||
|
||||
wake chan struct{}
|
||||
once sync.Once
|
||||
mu sync.Mutex
|
||||
asked bool
|
||||
watched map[string]bool
|
||||
refused map[string]string
|
||||
have map[string]string
|
||||
}
|
||||
|
||||
func (f *Follower) init() {
|
||||
f.once.Do(func() {
|
||||
f.wake = make(chan struct{}, 1)
|
||||
f.watched, f.refused, f.have = map[string]bool{}, map[string]string{}, map[string]string{}
|
||||
})
|
||||
}
|
||||
|
||||
// Wake asks for one more composition: after a change, and after a reconnect.
|
||||
func (f *Follower) Wake() {
|
||||
f.init()
|
||||
select {
|
||||
case f.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Refused is every watch refused at the last try, with why.
|
||||
func (f *Follower) Refused() map[string]string {
|
||||
f.init()
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := map[string]string{}
|
||||
for k, v := range f.refused {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *Follower) logf(format string, args ...any) {
|
||||
if f.Logf != nil {
|
||||
f.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the
|
||||
// bus module's loaded list moving.
|
||||
func (f *Follower) heard(state, key string, value []byte) bool {
|
||||
if state != ReadHave {
|
||||
return true
|
||||
}
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if isNode {
|
||||
if _, seen := f.have[node]; seen {
|
||||
return false
|
||||
}
|
||||
f.have[node] = ""
|
||||
return true
|
||||
}
|
||||
if module != BusModule {
|
||||
return false
|
||||
}
|
||||
var h struct {
|
||||
UserList json.RawMessage `json:"user-list"`
|
||||
}
|
||||
_ = json.Unmarshal(value, &h)
|
||||
was, seen := f.have[key]
|
||||
f.have[key] = string(h.UserList)
|
||||
return !seen || was != string(h.UserList)
|
||||
}
|
||||
|
||||
// open opens every watch not open yet; one refused is said and left for the next wake.
|
||||
func (f *Follower) open() {
|
||||
if f.Asked != nil && !f.asked {
|
||||
if err := f.Asked(f.Wake); err != nil {
|
||||
f.logf("a tool's ask to compose now cannot be heard yet (%v): tried again at the next change", err)
|
||||
} else {
|
||||
f.asked = true
|
||||
}
|
||||
}
|
||||
for _, state := range Followed {
|
||||
f.mu.Lock()
|
||||
open := f.watched[state]
|
||||
f.mu.Unlock()
|
||||
if open {
|
||||
continue
|
||||
}
|
||||
state := state
|
||||
err := f.Watch.Watch(state, func(key string, value []byte) {
|
||||
if f.heard(state, key, value) {
|
||||
f.Wake()
|
||||
}
|
||||
}, func(err error) {
|
||||
f.mu.Lock()
|
||||
f.watched[state] = false
|
||||
f.mu.Unlock()
|
||||
f.logf("the watch of %s ended (%v): opened again at the next change", state, err)
|
||||
f.Wake()
|
||||
})
|
||||
f.mu.Lock()
|
||||
if err != nil {
|
||||
f.refused[state] = err.Error()
|
||||
f.logf("%s cannot be watched yet (%v): composed without it, and tried again at the next change or reconnect", state, err)
|
||||
} else {
|
||||
f.watched[state] = true
|
||||
delete(f.refused, state)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Run composes at start and after every change until ctx ends. It returns only when ctx ends.
|
||||
func (f *Follower) Run(ctx context.Context) {
|
||||
f.init()
|
||||
f.Wake()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-f.wake:
|
||||
f.open()
|
||||
res := f.Part.Pass()
|
||||
for _, e := range res.Errors {
|
||||
f.logf("%s", e)
|
||||
}
|
||||
if len(res.Published) > 0 {
|
||||
f.logf("published %d memberships: %v", len(res.Published), res.Published)
|
||||
}
|
||||
if len(res.NotPublished) > 0 {
|
||||
f.logf("%d memberships not published, tried again at the next event; the first: %s", len(res.NotPublished), res.NotPublished[0])
|
||||
}
|
||||
if res.Written {
|
||||
f.logf("the shadow list moved to revision %d: %d users, %d differing from the loaded list",
|
||||
res.Revision, res.UserList.Users, len(res.UserList.Compare.Differing()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
// Package grants is the grants part in shadow (novox/hq ADR 0547 clauses 5 to 8, issue 1945): it composes the bus's user
|
||||
// list, each module's membership and each manifest's buckets, logs and streams from what the bus holds, with the bus
|
||||
// composer the controller uses (mesh-sdk buscompose), and compares them with what the bus loaded and holds. It writes only
|
||||
// its own state and takes no effect: it publishes no membership, asserts nothing, writes no grant and no user list the
|
||||
// bus node reads, until the operator's take-over, which this milestone always refuses.
|
||||
package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The states of other parts the grants part reads, as its manifest's reads name them (`<module>.<state>`).
|
||||
const (
|
||||
ReadAssignments = "mesh-controller.assignments"
|
||||
ReadControllerGrant = "mesh-controller.grants"
|
||||
ReadModules = "mesh-delivery-indexer.modules"
|
||||
ReadManifests = "mesh-delivery-indexer.manifests"
|
||||
ReadHave = "node-engine.have"
|
||||
ReadResolved = "mesh-config.resolved"
|
||||
ReadWants = "mesh-delivery-assigner.wants"
|
||||
ReadBuilds = "mesh-delivery-builder.builds"
|
||||
// ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9).
|
||||
ReadAccounts = "mesh-vault.bus-accounts"
|
||||
)
|
||||
|
||||
// Reads is every state the part reads, in the manifest's order.
|
||||
var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
|
||||
ReadAccounts}
|
||||
|
||||
// The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
|
||||
const (
|
||||
CallUsers = "nats.nats_users"
|
||||
CallUserCan = "nats.nats_user_can"
|
||||
CallStreams = "nats.nats_streams"
|
||||
)
|
||||
|
||||
// Invokes is every tool the part calls.
|
||||
var Invokes = []string{CallUsers, CallUserCan, CallStreams}
|
||||
|
||||
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
|
||||
// an error the part names as a gap and composes past: it never stops for one.
|
||||
type Reader interface {
|
||||
Keys(state string) ([]string, error)
|
||||
Get(state, key string) ([]byte, bool, error)
|
||||
}
|
||||
|
||||
// Caller calls another module's tool, `<module>.<tool>`, and answers its result.
|
||||
type Caller interface {
|
||||
Call(key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Gap is an input the shadow cannot read from the bus, named with why: never guessed.
|
||||
type Gap struct {
|
||||
Input string `json:"input"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// Standing gaps: what the controller composes from its own store, which no bus value carries in this milestone.
|
||||
var standingGaps = []Gap{
|
||||
{Input: "enrolment tokens", Why: "the live enrolment tokens are in the controller's store alone: no enrolment user is composed"},
|
||||
{Input: "people", Why: "the people and the tools they may invoke are in the controller's store alone: no person user is composed"},
|
||||
{Input: "seat holdings", Why: "who holds each seat held once for the mesh is in the controller's store alone: a mesh seat claimed on two machines is composed as held on both"},
|
||||
{Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"},
|
||||
}
|
||||
|
||||
// assignment is a key of the controller's assignments as composing needs it (mesh-controller internal/chain Assignment):
|
||||
// a value that names no time assigned (a module pulled in), has ended, or waits in a state, is no assignment.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Assigned time.Time `json:"assigned,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
State string `json:"state,omitempty"`
|
||||
}
|
||||
|
||||
func (a assignment) stands() bool { return !a.Assigned.IsZero() && a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// resolved is a key of mesh-config's resolved values, `<node>.<module>.<key>`.
|
||||
type resolved struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
// Inputs is what the shadow composes from, read from the bus.
|
||||
type Inputs struct {
|
||||
// Nodes is every machine seen: in a node agent's have or a standing assignment.
|
||||
Nodes []string
|
||||
// Assigned is each machine's standing assignments, sorted.
|
||||
Assigned map[string][]string
|
||||
// Manifests is each module's manifest at the indexer's trunk, by module.
|
||||
Manifests map[string]catalogue.Manifest
|
||||
// Digests is each module's manifest digest, by module.
|
||||
Digests map[string]string
|
||||
// Running is the manifest a machine is wanted to run, by `<node>.<module>`: the one the builder built at the
|
||||
// fingerprint the assigner's want names. An assignment with none is composed from the trunk's manifest.
|
||||
Running map[string]catalogue.Manifest
|
||||
// Values is each resolved config value, by `<node>/<module>/<key>`.
|
||||
Values map[string]any
|
||||
// BusNode is the machine running the bus's module, and Loaded what its node agent says it loaded.
|
||||
BusNode string
|
||||
Loaded *delivery.UserListWritten
|
||||
// Hashes are the vault's hashes of the bus accounts, by bus user (ReadAccounts); kept in memory, never written.
|
||||
Hashes map[string]HeldHash
|
||||
Gaps []Gap
|
||||
}
|
||||
|
||||
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said or written by
|
||||
// the part in shadow.
|
||||
type HeldHash struct {
|
||||
Hash string `json:"hash"`
|
||||
Generation int `json:"generation,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Adopted bool `json:"adopted,omitempty"`
|
||||
}
|
||||
|
||||
// readHashes reads the vault's bus-accounts; a refusal is a gap, and every user then has no hash held.
|
||||
func readHashes(r Reader, gaps *[]Gap) map[string]HeldHash {
|
||||
out := map[string]HeldHash{}
|
||||
for user, raw := range each(r, ReadAccounts, gaps) {
|
||||
var h HeldHash
|
||||
if json.Unmarshal(raw, &h) != nil || h.Hash == "" {
|
||||
*gaps = append(*gaps, Gap{Input: ReadAccounts + " " + user, Why: "holds no hash"})
|
||||
continue
|
||||
}
|
||||
out[user] = h
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BusModule is the bus's module: it holds mesh-broker.
|
||||
const BusModule = "nats"
|
||||
|
||||
// Manifests caches manifests by digest: a digest names one manifest for ever.
|
||||
type Manifests struct {
|
||||
mu sync.Mutex
|
||||
parsed map[string]catalogue.Manifest
|
||||
}
|
||||
|
||||
func (c *Manifests) get(r Reader, digest string) (catalogue.Manifest, error) {
|
||||
// The lock guards the map alone, never a read on the bus: a tool never waits on a pass's round trip.
|
||||
c.mu.Lock()
|
||||
m, ok := c.parsed[digest]
|
||||
c.mu.Unlock()
|
||||
if ok {
|
||||
return m, nil
|
||||
}
|
||||
raw, found, err := r.Get(ReadManifests, digest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
if !found {
|
||||
return catalogue.Manifest{}, fmt.Errorf("the indexer holds no manifest %s", digest)
|
||||
}
|
||||
v, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
m, err = catalogue.ParseManifest([]byte(v.JSON))
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, fmt.Errorf("the manifest %s is refused by mesh-sdk's reader: %w", digest, err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.parsed == nil {
|
||||
c.parsed = map[string]catalogue.Manifest{}
|
||||
}
|
||||
c.parsed[digest] = m
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// each reads every current value of a state; a refusal is a gap and an empty map.
|
||||
func each(r Reader, state string, gaps *[]Gap) map[string][]byte {
|
||||
keys, err := r.Keys(state)
|
||||
if err != nil {
|
||||
*gaps = append(*gaps, Gap{Input: state, Why: "refused or absent: " + err.Error()})
|
||||
return map[string][]byte{}
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
v, found, err := r.Get(state, k)
|
||||
if err != nil {
|
||||
*gaps = append(*gaps, Gap{Input: state + " " + k, Why: err.Error()})
|
||||
continue
|
||||
}
|
||||
if found {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ReadInputs reads what the shadow composes from. It never fails: what it cannot read is a gap.
|
||||
func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{},
|
||||
Running: map[string]catalogue.Manifest{},
|
||||
Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
|
||||
nodes := map[string]bool{}
|
||||
for _, raw := range each(r, ReadAssignments, &in.Gaps) {
|
||||
var a assignment
|
||||
if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" {
|
||||
continue
|
||||
}
|
||||
nodes[a.Node] = true
|
||||
in.Assigned[a.Node] = append(in.Assigned[a.Node], a.Module)
|
||||
if a.Module == BusModule {
|
||||
in.BusNode = a.Node
|
||||
}
|
||||
}
|
||||
haveKeys, err := r.Keys(ReadHave)
|
||||
if err != nil {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error()})
|
||||
}
|
||||
for _, k := range haveKeys {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode {
|
||||
nodes[node] = true
|
||||
}
|
||||
}
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "machines", Why: "read from the node agents' haves and the assignments: a machine with neither is not seen"})
|
||||
for n := range nodes {
|
||||
in.Nodes = append(in.Nodes, n)
|
||||
}
|
||||
sort.Strings(in.Nodes)
|
||||
if in.BusNode != "" {
|
||||
if raw, found, err := r.Get(ReadHave, in.BusNode+"."+BusModule); err != nil {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "the loaded list's have", Why: err.Error()})
|
||||
} else if found {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
in.Loaded = h.UserList
|
||||
}
|
||||
}
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
for node := range in.Assigned {
|
||||
sort.Strings(in.Assigned[node])
|
||||
for _, m := range in.Assigned[node] {
|
||||
wanted[m] = true
|
||||
}
|
||||
}
|
||||
modules := make([]string, 0, len(wanted))
|
||||
for m := range wanted {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
// The seats every module declares, across the whole catalogue: read every module the indexer holds, not only the
|
||||
// assigned ones, since a seat is declared by one module and held by another.
|
||||
indexed, err := r.Keys(ReadModules)
|
||||
if err != nil {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadModules, Why: "refused or absent: " + err.Error()})
|
||||
}
|
||||
for _, module := range indexed {
|
||||
raw, found, err := r.Get(ReadModules, module)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeModule(raw)
|
||||
if err == nil && v.Manifest == "" {
|
||||
err = fmt.Errorf("the indexer's value names no manifest digest")
|
||||
}
|
||||
if err != nil {
|
||||
if wanted[module] {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "manifest of " + module, Why: err.Error()})
|
||||
}
|
||||
continue
|
||||
}
|
||||
m, err := cache.get(r, v.Manifest)
|
||||
if err != nil {
|
||||
if wanted[module] {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "manifest of " + module, Why: err.Error()})
|
||||
}
|
||||
continue
|
||||
}
|
||||
in.Manifests[module] = m
|
||||
in.Digests[module] = v.Manifest
|
||||
}
|
||||
for _, module := range modules {
|
||||
if _, ok := in.Manifests[module]; !ok && !gapNamed(in.Gaps, "manifest of "+module) {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "manifest of " + module,
|
||||
Why: "assigned, and the indexer holds no manifest of it at the trunk (a module registered from outside the indexed repositories): composed with no authority"})
|
||||
}
|
||||
}
|
||||
// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
|
||||
var fromTrunk []string
|
||||
for key, raw := range each(r, ReadWants, &in.Gaps) {
|
||||
node, module, err := delivery.ParseWantKey(key)
|
||||
if err != nil || !slices.Contains(in.Assigned[node], module) {
|
||||
continue
|
||||
}
|
||||
w, err := delivery.DecodeWant(raw)
|
||||
if err != nil {
|
||||
fromTrunk = append(fromTrunk, node+"."+module)
|
||||
continue
|
||||
}
|
||||
if w.Removed || w.FP == "" {
|
||||
continue
|
||||
}
|
||||
bk, err := delivery.BuilderBuildKey(module, w.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
braw, found, err := r.Get(ReadBuilds, bk)
|
||||
if err != nil || !found {
|
||||
fromTrunk = append(fromTrunk, node+"."+module)
|
||||
continue
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(braw)
|
||||
if err != nil || b.Manifest == "" {
|
||||
fromTrunk = append(fromTrunk, node+"."+module)
|
||||
continue
|
||||
}
|
||||
if m, err := cache.get(r, b.Manifest); err == nil {
|
||||
in.Running[node+"."+module] = m
|
||||
} else {
|
||||
fromTrunk = append(fromTrunk, node+"."+module)
|
||||
}
|
||||
}
|
||||
if len(fromTrunk) > 0 {
|
||||
sort.Strings(fromTrunk)
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "wanted manifests", Why: "no build record names the manifest at the wanted fingerprint, so the trunk's is used: " + strings.Join(fromTrunk, ", ")})
|
||||
}
|
||||
in.Hashes = readHashes(r, &in.Gaps)
|
||||
for _, raw := range each(r, ReadResolved, &in.Gaps) {
|
||||
var v resolved
|
||||
if json.Unmarshal(raw, &v) != nil || v.Node == "" || v.Module == "" || v.Key == "" || len(v.Value) == 0 {
|
||||
continue
|
||||
}
|
||||
var value any
|
||||
if json.Unmarshal(v.Value, &value) == nil {
|
||||
in.Values[v.Node+"/"+v.Module+"/"+v.Key] = value
|
||||
}
|
||||
}
|
||||
sortGaps(in.Gaps)
|
||||
return in
|
||||
}
|
||||
|
||||
func gapNamed(gaps []Gap, input string) bool {
|
||||
for _, g := range gaps {
|
||||
if g.Input == input {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sortGaps(g []Gap) {
|
||||
sort.SliceStable(g, func(i, j int) bool { return strings.Compare(g[i].Input, g[j].Input) < 0 })
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// Memberships are published for real from this milestone (the operator, 2026-10-25 22:31): since release 6 nobody
|
||||
// publishes them, so a module placed since cannot serve its seat or reach a new state. The user list, the assertions and
|
||||
// the grants stay in shadow until take-over.
|
||||
|
||||
// MembershipGaps are what a membership carries that the part cannot compose from the bus.
|
||||
var MembershipGaps = []Gap{
|
||||
{Input: "membership: private network and received contributions", Why: "kept in each node's composition, which only that node reads: carried over from the membership last published, none for a first one"},
|
||||
{Input: "membership: root-free machines", Why: "judged by the controller on the bus at its composition, not kept: a verified sender is composed as from no root-free machine"},
|
||||
}
|
||||
|
||||
// Publisher publishes a membership on the assignments stream (membership.Subject), last per subject, and reads the one
|
||||
// last published there. A refusal (the grant not yet on the bus) is an error the part says and tries again at the next
|
||||
// event.
|
||||
type Publisher interface {
|
||||
LastMembership(node, module string) (body []byte, seq uint64, found bool, err error)
|
||||
PublishMembership(node, module string, body []byte, after uint64) error
|
||||
}
|
||||
|
||||
// Membership is one module's membership on one node, as composed now, and what became of its publishing.
|
||||
type Membership struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Digest string `json:"digest"`
|
||||
Membership membership.Membership `json:"membership"`
|
||||
// Published says what became of it at the last pass: "published", "unchanged" (equal to the one last published),
|
||||
// or why it was not.
|
||||
Published string `json:"published"`
|
||||
}
|
||||
|
||||
// Memberships composes every assignment's membership from the records (buscompose.MembershipFor, as the controller's
|
||||
// parked composition pass, mesh-controller 003eee8a, issued them), by `<node>.<module>`. Pure.
|
||||
func Memberships(r buscompose.Records) map[string]Membership {
|
||||
where := buscompose.PlacementsOf(r, r.Interchangeable)
|
||||
out := map[string]Membership{}
|
||||
for node, declared := range r.Assigned {
|
||||
for _, d := range declared {
|
||||
m := buscompose.MembershipFor(node, d, where)
|
||||
out[node+"."+d.Module] = Membership{Node: node, Module: d.Module, Digest: digestOf(m), Membership: m}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func digestOf(m membership.Membership) string {
|
||||
raw, _ := json.Marshal(m)
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Published words of a membership's last pass.
|
||||
const (
|
||||
MembershipPublished = "published"
|
||||
MembershipUnchanged = "unchanged"
|
||||
)
|
||||
|
||||
// publish publishes m when it differs from the membership last published for its assignment, carrying that one's private
|
||||
// network and received contributions; it answers m as published (or why not) and whether it was published now.
|
||||
func publish(p Publisher, m Membership) (Membership, bool) {
|
||||
if p == nil {
|
||||
m.Published = "not published: no publisher here (only the part's process publishes)"
|
||||
return m, false
|
||||
}
|
||||
was, seq, found, err := p.LastMembership(m.Node, m.Module)
|
||||
if err != nil {
|
||||
m.Published = "not published: the membership last published cannot be read: " + err.Error()
|
||||
return m, false
|
||||
}
|
||||
if found {
|
||||
var last membership.Membership
|
||||
if json.Unmarshal(was, &last) == nil {
|
||||
m.Membership.Mesh, m.Membership.Receives = last.Mesh, last.Receives
|
||||
m.Digest = digestOf(m.Membership)
|
||||
}
|
||||
}
|
||||
body, err := json.Marshal(m.Membership)
|
||||
if err != nil {
|
||||
m.Published = "not published: " + err.Error()
|
||||
return m, false
|
||||
}
|
||||
if found && sameMembership(was, body) {
|
||||
m.Published = MembershipUnchanged
|
||||
return m, false
|
||||
}
|
||||
// Only a first membership, or one that only adds to the last, is published (the lead's decision, review of 1ff5049):
|
||||
// composed with no root-free judgement, a kind's verified-sender would be dropped, and the operator's approvals by
|
||||
// a channel would no longer be honoured. Any loss stays a shadow difference, named.
|
||||
if found {
|
||||
if lost := Losses(was, body); len(lost) > 0 {
|
||||
m.Published = ShadowDifference + strings.Join(lost, "; ")
|
||||
return m, false
|
||||
}
|
||||
}
|
||||
if err := p.PublishMembership(m.Node, m.Module, body, seq); err != nil {
|
||||
m.Published = fmt.Sprintf("not published (tried again at the next event): %v", err)
|
||||
return m, false
|
||||
}
|
||||
m.Published = MembershipPublished
|
||||
return m, true
|
||||
}
|
||||
|
||||
// sameMembership says two memberships say the same, whatever the order of their fields. Pure.
|
||||
func sameMembership(was, now []byte) bool {
|
||||
var a, b any
|
||||
if json.Unmarshal(was, &a) != nil || json.Unmarshal(now, &b) != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// ShadowDifference opens what a membership's pass says when it was not published because it would lose something.
|
||||
const ShadowDifference = "not published, a shadow difference: publishing would lose "
|
||||
|
||||
// Losses names everything the last membership holds that now drops or weakens: a field gone, a value changed, a list
|
||||
// element (a capability, a kind, a served address, a seat, a state, a stream) with no element in now that holds all of it.
|
||||
// Empty means now holds all the last one did, and maybe more. Pure.
|
||||
func Losses(last, now []byte) []string {
|
||||
var a, b any
|
||||
if json.Unmarshal(last, &a) != nil {
|
||||
return nil
|
||||
}
|
||||
if json.Unmarshal(now, &b) != nil {
|
||||
return []string{"the new membership is not JSON"}
|
||||
}
|
||||
return losses("", a, b)
|
||||
}
|
||||
|
||||
func losses(path string, last, now any) []string {
|
||||
switch l := last.(type) {
|
||||
case map[string]any:
|
||||
n, ok := now.(map[string]any)
|
||||
if !ok {
|
||||
return []string{fmt.Sprintf("%s: %s", name(path), brief(last))}
|
||||
}
|
||||
keys := make([]string, 0, len(l))
|
||||
for k := range l {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []string
|
||||
for _, k := range keys {
|
||||
nv, ok := n[k]
|
||||
if !ok {
|
||||
out = append(out, fmt.Sprintf("%s: %s", name(path+"."+k), brief(l[k])))
|
||||
continue
|
||||
}
|
||||
out = append(out, losses(path+"."+k, l[k], nv)...)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
n, _ := now.([]any)
|
||||
var out []string
|
||||
for i, e := range l {
|
||||
if !slices.ContainsFunc(n, func(c any) bool { return len(losses("", e, c)) == 0 }) {
|
||||
out = append(out, fmt.Sprintf("%s[%d]: %s", name(path), i, brief(e)))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
if !reflect.DeepEqual(last, now) {
|
||||
return []string{fmt.Sprintf("%s: %s, now %s", name(path), brief(last), brief(now))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func name(path string) string {
|
||||
if path == "" {
|
||||
return "the membership"
|
||||
}
|
||||
return strings.TrimPrefix(path, ".")
|
||||
}
|
||||
|
||||
// brief is a value in JSON, cut at 200 bytes.
|
||||
func brief(v any) string {
|
||||
raw, _ := json.Marshal(v)
|
||||
if len(raw) > 200 {
|
||||
return string(raw[:200]) + "…"
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The part's own states, as its manifest declares them, and the keys of the two that hold one value.
|
||||
const (
|
||||
StateUserList = "user-list"
|
||||
StateMemberships = "memberships"
|
||||
StateAssertions = "assertions"
|
||||
StateTakeover = "takeover"
|
||||
KeyList = "list"
|
||||
KeySwitch = "switch"
|
||||
// KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList.
|
||||
KeyPass = "pass"
|
||||
KeyAsked = "asked"
|
||||
)
|
||||
|
||||
// States is every state the part keeps.
|
||||
var States = []string{StateAssertions, StateMemberships, StateTakeover, StateUserList}
|
||||
|
||||
// The events the part says.
|
||||
const (
|
||||
EventUserList = "user-list-changed"
|
||||
EventMembership = "membership-changed"
|
||||
)
|
||||
|
||||
// Emits is every event the part says.
|
||||
var Emits = []string{EventMembership, EventUserList}
|
||||
|
||||
// Shadow is the switch's one state in this milestone: the bus node follows the controller's list.
|
||||
const Shadow = "shadow"
|
||||
|
||||
// Store is the part's own state, by local name: the only thing the part writes.
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, uint64, bool, error)
|
||||
Put(state, key string, value []byte) (uint64, error)
|
||||
}
|
||||
|
||||
// Emitter says one of the part's events.
|
||||
type Emitter interface {
|
||||
Emit(event string, body any) error
|
||||
}
|
||||
|
||||
// UserListState is the value of `user-list` in shadow: the list the grants part composed (users and grants, no hashes),
|
||||
// what the bus loaded, and how they differ. Inputs is the digest of all it says but At: a value is written again only
|
||||
// when it changes.
|
||||
type UserListState struct {
|
||||
V int `json:"v"`
|
||||
Mode string `json:"mode"`
|
||||
Inputs string `json:"inputs"`
|
||||
Digest string `json:"digest"`
|
||||
Users int `json:"users"`
|
||||
// LeftOut are the assignments the shadow list leaves out, each with why.
|
||||
LeftOut []Gap `json:"left_out"`
|
||||
Compare Comparison `json:"compare"`
|
||||
Loaded *delivery.UserListWritten `json:"loaded,omitempty"`
|
||||
// Credentials says per user whether the vault holds its hash and how its line would differ (ADR 0547 clause 9).
|
||||
Credentials Credentials `json:"credentials"`
|
||||
BusNode string `json:"bus_node,omitempty"`
|
||||
Gaps []Gap `json:"gaps"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
func (s UserListState) inputs() string {
|
||||
s.At, s.Inputs = time.Time{}, ""
|
||||
raw, _ := json.Marshal(s)
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Result is what one composition did.
|
||||
type Result struct {
|
||||
UserList UserListState `json:"user_list"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Written bool `json:"written"`
|
||||
Memberships int `json:"memberships"`
|
||||
MembershipsWritten int `json:"memberships_written"`
|
||||
Published []string `json:"published,omitempty"`
|
||||
NotPublished []string `json:"not_published,omitempty"`
|
||||
Assertions int `json:"assertions"`
|
||||
AssertionsWritten int `json:"assertions_written"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
PublishedOrAsserted string `json:"published_or_asserted"`
|
||||
}
|
||||
|
||||
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
|
||||
const NothingTakesEffect = "memberships only (on the assignments stream, a first one or one that only adds to the last published; a loss stays a shadow difference); the user list, assertions and grants stay in shadow until take-over"
|
||||
|
||||
// Part is the grants part: it reads, calls and writes through these, and nothing else.
|
||||
type Part struct {
|
||||
Read Reader
|
||||
Call Caller
|
||||
Store Store
|
||||
Emit Emitter
|
||||
// Publish publishes memberships: the process's connection; nil in the tools, which ask the process instead.
|
||||
Publish Publisher
|
||||
Now func() time.Time
|
||||
|
||||
cache Manifests
|
||||
mu sync.Mutex
|
||||
last Result
|
||||
}
|
||||
|
||||
func (p *Part) now() time.Time {
|
||||
if p.Now == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return p.Now().UTC()
|
||||
}
|
||||
|
||||
// Compose composes the user list, the memberships and the assertions from what the bus holds now, compares them, and
|
||||
// writes each of its own keys whose value changed, saying each change. It never stops at a refusal: a read refused is
|
||||
// a gap, a write refused is said in Errors, and the next event composes again.
|
||||
func (p *Part) Compose() Result {
|
||||
// No lock is held across a round trip on the bus: the reads, calls, publishes and writes below take none, and the
|
||||
// lock is taken only to keep the result, so a tool answers while a pass waits on the bus.
|
||||
res := Result{PublishedOrAsserted: NothingTakesEffect}
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
p.last = res
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
in := ReadInputs(p.Read, &p.cache)
|
||||
recs, left := Records(in)
|
||||
st := UserListState{V: 1, Mode: Shadow, LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
|
||||
Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}
|
||||
if st.LeftOut == nil {
|
||||
st.LeftOut = []Gap{}
|
||||
}
|
||||
users, err := ComposeUsers(recs)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
|
||||
}
|
||||
users = WithHashes(users, in.Hashes)
|
||||
st.Digest, st.Users = Digest(users), len(users)
|
||||
loaded, err := LoadedUsers(p.Call)
|
||||
st.Compare = Compare(users, loaded, Placed(in))
|
||||
if err != nil {
|
||||
st.Compare.Unread = err.Error()
|
||||
}
|
||||
st.Credentials = CredentialsOf(users, loaded, in.Hashes)
|
||||
st.Inputs = st.inputs()
|
||||
res.UserList = st
|
||||
if raw, rev, found, err := p.Store.Get(StateUserList, KeyList); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be read: %v", StateUserList, err))
|
||||
} else {
|
||||
var was UserListState
|
||||
if found && json.Unmarshal(raw, &was) == nil && was.Inputs == st.Inputs {
|
||||
res.Revision = rev
|
||||
} else if data, err := json.Marshal(st); err == nil {
|
||||
if rev, err := p.Store.Put(StateUserList, KeyList, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be written: %v", StateUserList, err))
|
||||
} else {
|
||||
res.Revision, res.Written = rev, true
|
||||
p.say(&res, EventUserList, map[string]any{"digest": st.Digest, "revision": rev, "users": st.Users,
|
||||
"differing": len(st.Compare.Differing()), "mode": Shadow})
|
||||
}
|
||||
}
|
||||
}
|
||||
ms := Memberships(recs)
|
||||
res.Memberships = len(ms)
|
||||
for _, key := range sortedKeys(ms) {
|
||||
m, now := publish(p.Publish, ms[key])
|
||||
switch {
|
||||
case now:
|
||||
res.Published = append(res.Published, key)
|
||||
p.say(&res, EventMembership, map[string]any{"node": m.Node, "module": m.Module, "digest": m.Digest, "published": true})
|
||||
case m.Published != MembershipUnchanged:
|
||||
res.NotPublished = append(res.NotPublished, key+": "+m.Published)
|
||||
}
|
||||
data, _ := json.Marshal(m)
|
||||
if p.changed(&res, StateMemberships, key, data) {
|
||||
res.MembershipsWritten++
|
||||
}
|
||||
}
|
||||
held, heldErr := HeldStreams(p.Call)
|
||||
for _, module := range sortedKeys(in.Manifests) {
|
||||
if !assigned(in, module) {
|
||||
continue
|
||||
}
|
||||
a := Assert(module, in.Digests[module], in.Manifests[module], held, heldErr)
|
||||
res.Assertions++
|
||||
data, _ := json.Marshal(a)
|
||||
if p.changed(&res, StateAssertions, module, data) {
|
||||
res.AssertionsWritten++
|
||||
}
|
||||
}
|
||||
if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found {
|
||||
data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}})
|
||||
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be written: %v", StateTakeover, err))
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Last is the result of the last pass this part made, kept under the lock alone.
|
||||
func (p *Part) Last() Result {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.last
|
||||
}
|
||||
|
||||
func assigned(in Inputs, module string) bool {
|
||||
for _, ms := range in.Assigned {
|
||||
for _, m := range ms {
|
||||
if m == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// changed writes key when its value differs from the one held, and says whether it wrote.
|
||||
func (p *Part) changed(res *Result, state, key string, data []byte) bool {
|
||||
raw, _, found, err := p.Store.Get(state, key)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", state, key, err))
|
||||
return false
|
||||
}
|
||||
if found && bytes.Equal(raw, data) {
|
||||
return false
|
||||
}
|
||||
if _, err := p.Store.Put(state, key, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", state, key, err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Part) say(res *Result, event string, body any) {
|
||||
if p.Emit == nil {
|
||||
return
|
||||
}
|
||||
if err := p.Emit.Emit(event, body); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("the event %s could not be said: %v", event, err))
|
||||
}
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Switch is the value of `takeover`: which list the bus node follows, and every act on the switch.
|
||||
type Switch struct {
|
||||
V int `json:"v"`
|
||||
State string `json:"state"`
|
||||
Acts []Act `json:"acts"`
|
||||
}
|
||||
|
||||
// Act is one take-over or hand-back asked, done or refused.
|
||||
type Act struct {
|
||||
Act string `json:"act"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// keptActs is how many acts the switch keeps, newest last.
|
||||
const keptActs = 50
|
||||
|
||||
// Operator says whether a caller is the operator's credential: a person's own bus account (`person.<name>`), which no
|
||||
// session of an agent holds (an agent calls as its machine's tool runner).
|
||||
func Operator(caller string) bool {
|
||||
name, ok := strings.CutPrefix(caller, "person.")
|
||||
return ok && name != ""
|
||||
}
|
||||
|
||||
// NotComposable is why take-over is refused in this milestone, whatever the lists say.
|
||||
const NotComposable = "the enrolment users and the persons cannot be composed: they live only in the controller's store, and take-over waits until they have records on the bus (ADR 0547 clause 9, version 26)"
|
||||
|
||||
// record keeps an act on the switch; a refused write is said in the answer.
|
||||
func (p *Part) record(a Act) string {
|
||||
raw, _, found, err := p.Store.Get(StateTakeover, KeySwitch)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("; the act is not recorded: %v", err)
|
||||
}
|
||||
s := Switch{V: 1, State: Shadow}
|
||||
if found {
|
||||
_ = json.Unmarshal(raw, &s)
|
||||
}
|
||||
s.Acts = append(s.Acts, a)
|
||||
if len(s.Acts) > keptActs {
|
||||
s.Acts = s.Acts[len(s.Acts)-keptActs:]
|
||||
}
|
||||
data, _ := json.Marshal(s)
|
||||
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
|
||||
return fmt.Sprintf("; the act is not recorded: %v", err)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TakeOver is the operator's switch to the grants part's list. It is refused without the operator's credential, while
|
||||
// any user of the shadow list differs from the loaded list (but those both agree are gone), and in this milestone
|
||||
// always, since the enrolment users and the persons cannot be composed. Every ask is recorded.
|
||||
func (p *Part) TakeOver(caller, why string) error {
|
||||
a := Act{Act: "take-over", By: caller, Why: why, At: p.now()}
|
||||
if !Operator(caller) {
|
||||
a.Refused = fmt.Sprintf("take-over is the operator's alone: %s is not a person's account", caller)
|
||||
return fmt.Errorf("%s%s", a.Refused, p.record(a))
|
||||
}
|
||||
res := p.Compose()
|
||||
var reasons []string
|
||||
if c := res.UserList.Compare; !c.Equal() {
|
||||
if c.Unread != "" {
|
||||
reasons = append(reasons, "the loaded list could not be read: "+c.Unread)
|
||||
}
|
||||
if d := c.Differing(); len(d) > 0 {
|
||||
reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list: %s", len(d), strings.Join(d, ", ")))
|
||||
}
|
||||
}
|
||||
cr, gone := res.UserList.Credentials, res.UserList.Compare.Gone
|
||||
var unhashed []string
|
||||
for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) {
|
||||
if !slices.Contains(gone, u) {
|
||||
unhashed = append(unhashed, u)
|
||||
}
|
||||
}
|
||||
if len(unhashed) > 0 {
|
||||
sort.Strings(unhashed)
|
||||
reasons = append(reasons, fmt.Sprintf("the vault holds no hash for %d users (ADR 0547 clause 9: handed or adopted before take-over): %s",
|
||||
len(unhashed), strings.Join(unhashed, ", ")))
|
||||
}
|
||||
reasons = append(reasons, NotComposable)
|
||||
a.Refused = "take-over refused: " + strings.Join(reasons, "; ")
|
||||
return fmt.Errorf("%s%s", a.Refused, p.record(a))
|
||||
}
|
||||
|
||||
// HandBack is the operator's switch back to the controller's list. Refused without the operator's credential, and
|
||||
// while the part is in shadow, since the bus node follows the controller's list already. Every ask is recorded.
|
||||
func (p *Part) HandBack(caller, why string) error {
|
||||
a := Act{Act: "hand-back", By: caller, Why: why, At: p.now()}
|
||||
if !Operator(caller) {
|
||||
a.Refused = fmt.Sprintf("hand-back is the operator's alone: %s is not a person's account", caller)
|
||||
return fmt.Errorf("%s%s", a.Refused, p.record(a))
|
||||
}
|
||||
a.Refused = "nothing to hand back: the grants part is in shadow and never took over, so the bus node follows the controller's list already"
|
||||
return fmt.Errorf("%s%s", a.Refused, p.record(a))
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Records is what the bus composer composes from, rebuilt from the bus (mesh-controller internal/inventory busrecords.go,
|
||||
// BusRecords, read the same way from Inputs instead of the controller's store). Pure. An assigned module whose manifest
|
||||
// the indexer does not hold is left out and named, where the controller refuses the whole composition: the shadow
|
||||
// composes what it can and says what it could not.
|
||||
func Records(in Inputs) (buscompose.Records, []Gap) {
|
||||
var gaps []Gap
|
||||
seats := map[string]catalogue.SeatDeclaration{}
|
||||
declarers := map[string]string{}
|
||||
for _, m := range in.Manifests {
|
||||
for _, s := range m.DefinesSeats {
|
||||
seats[s.Name] = s
|
||||
declarers[s.Name] = m.Module
|
||||
}
|
||||
}
|
||||
for _, own := range catalogue.SeatsWithAProtocol() {
|
||||
seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves}
|
||||
}
|
||||
out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{},
|
||||
Interchangeable: map[string]bool{}}
|
||||
values := func(provider, setting string) []any {
|
||||
var vs []any
|
||||
for _, node := range in.Nodes {
|
||||
if v, ok := in.Values[node+"/"+provider+"/"+setting]; ok && slices.Contains(in.Assigned[node], provider) {
|
||||
vs = append(vs, v)
|
||||
}
|
||||
}
|
||||
return vs
|
||||
}
|
||||
for _, node := range in.Nodes {
|
||||
out.Nodes = append(out.Nodes, node)
|
||||
manifestOf := func(module string) (catalogue.Manifest, bool) {
|
||||
if m, ok := in.Running[node+"."+module]; ok {
|
||||
return m, true
|
||||
}
|
||||
m, ok := in.Manifests[module]
|
||||
return m, ok
|
||||
}
|
||||
var here []catalogue.Manifest
|
||||
for _, module := range in.Assigned[node] {
|
||||
if m, ok := manifestOf(module); ok {
|
||||
here = append(here, m)
|
||||
}
|
||||
}
|
||||
for _, module := range in.Assigned[node] {
|
||||
m, known := manifestOf(module)
|
||||
if !known {
|
||||
gaps = append(gaps, Gap{Input: module + " on " + node, Why: "left out of the shadow list: no manifest to derive what it may say from"})
|
||||
continue
|
||||
}
|
||||
d := declaredFor(m, seats, declarers)
|
||||
for _, sr := range catalogue.ReadsGranted(m, here, node) {
|
||||
d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key})
|
||||
}
|
||||
d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes)
|
||||
out.Assigned[node] = append(out.Assigned[node], d)
|
||||
if m.Instances == catalogue.InstancesInterchangeable {
|
||||
out.Interchangeable[m.Module] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, gaps
|
||||
}
|
||||
|
||||
// declaredFor is mesh-controller's declaredFor (internal/inventory/busrecords.go), copied: one module's manifest as the
|
||||
// composer needs it.
|
||||
func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration, declarers map[string]string) buscompose.Declared {
|
||||
var fromModules []string
|
||||
var watches []buscompose.Seat
|
||||
for _, c := range m.Consumes {
|
||||
emitter, event, named := strings.Cut(c, ".")
|
||||
if named {
|
||||
if s, isASeat := seats[emitter]; isASeat && s.Kinded {
|
||||
ev := strings.TrimSuffix(event, ".*")
|
||||
if catalogue.SeatSays(s.Emits, ev) {
|
||||
watches = append(watches, buscompose.Seat{Name: s.Name, Scope: s.Scope, Emits: []string{ev},
|
||||
Kinded: true, Proofs: s.Proofs, DeclaredBy: declarers[s.Name]})
|
||||
continue
|
||||
}
|
||||
}
|
||||
if s, isASeat := seats[emitter]; isASeat && catalogue.SeatSays(s.Emits, event) {
|
||||
watches = append(watches, buscompose.Seat{Name: s.Name, Emits: []string{event}})
|
||||
continue
|
||||
}
|
||||
}
|
||||
fromModules = append(fromModules, c)
|
||||
}
|
||||
d := buscompose.Declared{
|
||||
Module: m.Module, Emits: m.EmitsAll(), Consumes: fromModules, Watches: watches, Serves: m.Tools,
|
||||
Invokes: m.Invokes, Waits: waitsOf(m, seats), State: bucketsOf(m), Reads: m.Reads, Logs: logsOf(m),
|
||||
Streams: streamsOf(m), Queues: queueOwnersOf(m), Processes: processesOf(m), Checks: catalogue.HealthChecks(m),
|
||||
RunsAs: m.RunsAs, Issued: issuedTools(m),
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
d.Loads = true
|
||||
}
|
||||
}
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
d.NoAccount = true
|
||||
}
|
||||
if _, reads := m.OwnSecrets[buscompose.ProvisionerSecret]; reads && m.ClaimsSeat(buscompose.NodeProvisionerSeat) {
|
||||
d.ProvisionsNode = true
|
||||
}
|
||||
d.SnapshotsTheBus = m.ClaimsSeat(catalogue.BrokerSeat)
|
||||
for _, c := range m.Claims {
|
||||
if s, hasAProtocol := seats[c.Name]; hasAProtocol {
|
||||
held := asSeat(s, declarers[s.Name])
|
||||
held.Kind = c.Kind
|
||||
held.Capabilities = c.Capabilities
|
||||
d.Holds = append(d.Holds, held)
|
||||
}
|
||||
}
|
||||
for _, name := range m.Uses {
|
||||
if s, declaredSomewhere := seats[name]; declaredSomewhere {
|
||||
d.Uses = append(d.Uses, asSeat(s, declarers[s.Name]))
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func waitsOf(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) []buscompose.Wait {
|
||||
var out []buscompose.Wait
|
||||
for _, w := range m.Waits {
|
||||
emitter, event, _ := strings.Cut(w, ".")
|
||||
first, _, _ := strings.Cut(event, ".")
|
||||
wait := buscompose.Wait{Name: w, Emitter: emitter, Event: event}
|
||||
if s, isASeat := seats[emitter]; isASeat {
|
||||
switch {
|
||||
case s.Kinded && catalogue.SeatSays(s.Emits, first):
|
||||
wait.Seat, wait.Last = true, "kind"
|
||||
case s.NamedByCaller(first) && catalogue.SeatSays(s.Emits, first):
|
||||
wait.Seat, wait.Last = true, "asker"
|
||||
case catalogue.SeatSays(s.Emits, event):
|
||||
wait.Seat = true
|
||||
}
|
||||
}
|
||||
out = append(out, wait)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func issuedTools(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, o := range m.Provides {
|
||||
if tool, issued := m.IssuedCredentialOf(o.Name); issued && !slices.Contains(out, tool) {
|
||||
out = append(out, tool)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func bucketsOf(m catalogue.Manifest) []buscompose.Bucket {
|
||||
var out []buscompose.Bucket
|
||||
for _, s := range m.State {
|
||||
out = append(out, buscompose.Bucket{Module: m.Module, Name: s.Name, History: s.History, TTLSeconds: s.TTLSeconds,
|
||||
PerMachine: s.PerMachine, KeyTTL: s.KeyTTL})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func logsOf(m catalogue.Manifest) []buscompose.Log {
|
||||
var out []buscompose.Log
|
||||
for _, l := range m.Logs {
|
||||
out = append(out, buscompose.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func asSeat(s catalogue.SeatDeclaration, declarer string) buscompose.Seat {
|
||||
seat := buscompose.Seat{Name: s.Name, Scope: s.Scope, Accepts: s.Accepts, Emits: s.Emits,
|
||||
Serves: catalogue.VerbNames(s.Serves), Kinded: s.Kinded, ByCaller: s.ByCaller, Proofs: s.Proofs, DeclaredBy: declarer}
|
||||
if own, ok := catalogue.SeatNamed(s.Name); ok && own.PulledByKind {
|
||||
seat.PulledByKind = true
|
||||
}
|
||||
for _, r := range s.Records {
|
||||
if declarer != "" {
|
||||
seat.Records = append(seat.Records, buscompose.BucketName(declarer, r))
|
||||
}
|
||||
}
|
||||
return seat
|
||||
}
|
||||
|
||||
func streamsOf(m catalogue.Manifest) []buscompose.ModuleStream {
|
||||
var out []buscompose.ModuleStream
|
||||
for _, s := range m.Streams {
|
||||
events := make([]string, 0, len(s.Events))
|
||||
for _, e := range s.Events {
|
||||
events = append(events, buscompose.WirePattern(e))
|
||||
}
|
||||
out = append(out, buscompose.ModuleStream{Module: m.Module, Name: s.BusName(), Events: events,
|
||||
MaxAgeDays: s.Days(), MaxMiB: s.Cap(), Restricted: s.Grant != nil,
|
||||
WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue, Duplicates: s.Duplicates})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func queueOwnersOf(m catalogue.Manifest) []buscompose.QueueOwner {
|
||||
buses := m.ProcessBuses()
|
||||
if len(buses) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := map[string]catalogue.StreamDeclaration{}
|
||||
for _, s := range m.Streams {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
var out []buscompose.QueueOwner
|
||||
for _, r := range m.Resources {
|
||||
id := fmt.Sprint(r["id"])
|
||||
b, declared := buses[id]
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
for _, q := range b.Queues {
|
||||
s, known := byName[q]
|
||||
if !known || s.Retention != catalogue.StreamRetentionWorkQueue {
|
||||
continue
|
||||
}
|
||||
out = append(out, buscompose.QueueOwner{Process: id, Stream: s.BusName(), MaxDeliver: s.MaxDeliveries})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func processesOf(m catalogue.Manifest) []buscompose.ProcessPrincipal {
|
||||
buses := m.ProcessBuses()
|
||||
if len(buses) == 0 {
|
||||
return nil
|
||||
}
|
||||
queues := queueOwnersOf(m)
|
||||
var out []buscompose.ProcessPrincipal
|
||||
for _, r := range m.Resources {
|
||||
id := fmt.Sprint(r["id"])
|
||||
b, declared := buses[id]
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
pr := buscompose.ProcessPrincipal{Process: id, Emits: b.Emits, State: b.State, Asks: b.Asks, Invokes: b.Invokes}
|
||||
for _, q := range queues {
|
||||
if q.Process == id {
|
||||
pr.Queues = append(pr.Queues, q)
|
||||
}
|
||||
}
|
||||
out = append(out, pr)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hearsOf(m catalogue.Manifest, declared map[string]catalogue.Manifest, node string, values func(provider, setting string) []any,
|
||||
consumes []string) ([]buscompose.Heard, bool, []string) {
|
||||
reading := m
|
||||
reading.Consumes = consumes
|
||||
parts, touches, onEvents := catalogue.Hearing(reading, declared, node, values)
|
||||
var out []buscompose.Heard
|
||||
for _, p := range parts {
|
||||
h := buscompose.Heard{Stream: p.Stream.BusName(), Emitter: p.Emitter, Restricted: p.Stream.Grant != nil}
|
||||
for _, pattern := range p.Patterns {
|
||||
h.Filters = append(h.Filters, buscompose.ModuleStreamSubject(p.Emitter, buscompose.WirePattern(pattern)))
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, touches, onEvents
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The part's tools, as its manifest lists them. Module tools in this milestone: a `mesh-*` seat must be in mesh-sdk's own
|
||||
// seat list, which the running controller's composer does not know, so the part claims no seat until take-over.
|
||||
const (
|
||||
ToolGrants = "grants_grants"
|
||||
ToolMembership = "grants_membership"
|
||||
ToolUserList = "grants_user_list"
|
||||
ToolComposeNow = "grants_compose_now"
|
||||
ToolAssert = "grants_assert"
|
||||
ToolTakeOver = "grants_take_over"
|
||||
ToolHandBack = "grants_hand_back"
|
||||
)
|
||||
|
||||
// Tools is every tool, in the manifest's order.
|
||||
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack}
|
||||
|
||||
// WritesNoGrant is what grants says of the part's own grants in this milestone.
|
||||
const WritesNoGrant = "none: the grants part writes no grants.<module>.<fp> before ADR 0547's third milestone (clause 5); the controller's are beside"
|
||||
|
||||
// Grants answers the tool grants: what the part writes of grants (none), and the controller's, one module's or all.
|
||||
func (p *Part) Grants(module string) (any, error) {
|
||||
keys, err := p.Read.Keys(ReadControllerGrant)
|
||||
if err != nil {
|
||||
return map[string]any{"writes": WritesNoGrant, "controller_unread": err.Error()}, nil
|
||||
}
|
||||
type grant struct {
|
||||
Module string `json:"module"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Accounts []string `json:"accounts,omitempty"`
|
||||
}
|
||||
out := []grant{}
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseControllerGrantKey(k)
|
||||
if err != nil || (module != "" && m != module) {
|
||||
continue
|
||||
}
|
||||
g := grant{Module: m, Fingerprint: fp}
|
||||
if module != "" {
|
||||
if raw, found, err := p.Read.Get(ReadControllerGrant, k); err == nil && found {
|
||||
if v, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
g.Accounts = v.Accounts
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Fingerprint < out[j].Module+out[j].Fingerprint })
|
||||
return map[string]any{"writes": WritesNoGrant, "controller": out, "count": len(out)}, nil
|
||||
}
|
||||
|
||||
// MembershipOf answers the tool membership: one module on one node as composed now, and as last published by the
|
||||
// part's process (its state memberships says what became of each at the last pass).
|
||||
func (p *Part) MembershipOf(node, module string) (any, error) {
|
||||
if node == "" || module == "" {
|
||||
return nil, errors.New("membership names a node and a module")
|
||||
}
|
||||
in := ReadInputs(p.Read, &p.cache)
|
||||
recs, left := Records(in)
|
||||
m, ok := Memberships(recs)[node+"."+module]
|
||||
out := map[string]any{"node": node, "module": module, "gaps": MembershipGaps}
|
||||
if ok {
|
||||
out["composed"] = m
|
||||
} else {
|
||||
why := fmt.Sprintf("%s is not assigned to %s on the bus", module, node)
|
||||
for _, g := range left {
|
||||
if g.Input == module+" on "+node {
|
||||
why = g.Why
|
||||
}
|
||||
}
|
||||
out["composed"] = why
|
||||
}
|
||||
if raw, rev, found, err := p.Store.Get(StateMemberships, node+"."+module); err != nil {
|
||||
out["published"] = "not readable: " + err.Error()
|
||||
} else if found {
|
||||
out["published"] = map[string]any{"revision": rev, "last_pass": json.RawMessage(raw)}
|
||||
} else {
|
||||
out["published"] = "no pass of the part's process has composed it yet"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Pass composes and keeps the result as the last pass, for the tools to read.
|
||||
func (p *Part) Pass() Result {
|
||||
res := p.Compose()
|
||||
data, _ := json.Marshal(res)
|
||||
if _, err := p.Store.Put(StateUserList, KeyPass, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyPass, err))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// NoGrantAgain is what compose-now says of the controller's grants: the controller serves no verb that writes a module's
|
||||
// grant again (its issue replaces the account, which is not that), so a grant write it lost stays lost until it composes
|
||||
// again; the part writes grants only from ADR 0547's third milestone.
|
||||
const NoGrantAgain = "missing: a verb to ask the controller to write a module's grant again (mesh-controller serves none; its issue replaces the bus account, which is not that), so compose-now cannot ask it"
|
||||
|
||||
// AskCompose is the tool compose-now: it asks the part's process to compose and publish now (only the process holds the
|
||||
// grant to publish a membership), and answers the last pass, whose revision moves when the process answered.
|
||||
func (p *Part) AskCompose(caller string) (any, error) {
|
||||
ask, _ := json.Marshal(map[string]any{"by": caller, "at": p.now()})
|
||||
rev, err := p.Store.Put(StateUserList, KeyAsked, ask)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written: %w", err)
|
||||
}
|
||||
out := map[string]any{"asked": rev, "says": "the part's process composes and publishes at this ask; grants_user_list's last pass says what it published",
|
||||
"controller_grants": NoGrantAgain}
|
||||
if raw, prev, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
|
||||
out["last_pass"] = map[string]any{"revision": prev, "result": json.RawMessage(raw)}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UserList answers the tool user-list: the shadow list as last written, beside what the bus node says it loaded now.
|
||||
func (p *Part) UserList() (any, error) {
|
||||
out := map[string]any{"mode": Shadow}
|
||||
raw, rev, found, err := p.Store.Get(StateUserList, KeyList)
|
||||
switch {
|
||||
case err != nil:
|
||||
out["written"] = "not readable: " + err.Error()
|
||||
case !found:
|
||||
out["written"] = "nothing composed yet: grants_compose_now composes it"
|
||||
default:
|
||||
var st UserListState
|
||||
if err := json.Unmarshal(raw, &st); err != nil {
|
||||
return nil, fmt.Errorf("its own user list cannot be read: %w", err)
|
||||
}
|
||||
out["revision"], out["written"] = rev, st
|
||||
if raw, prev, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
|
||||
out["last_pass"] = map[string]any{"revision": prev, "result": json.RawMessage(raw)}
|
||||
}
|
||||
if st.BusNode != "" {
|
||||
if raw, found, err := p.Read.Get(ReadHave, st.BusNode+"."+BusModule); err == nil && found {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
out["loaded_now"] = h.UserList
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AssertOne answers the tool assert: one module's buckets, logs and streams as composed from its manifest at the
|
||||
// indexer's trunk, beside what the bus holds. It asserts nothing.
|
||||
func (p *Part) AssertOne(module string) (any, error) {
|
||||
if module == "" {
|
||||
return nil, errors.New("assert names a module")
|
||||
}
|
||||
raw, found, err := p.Read.Get(ReadModules, module)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the indexer holds no module %s", module)
|
||||
}
|
||||
v, err := delivery.DecodeModule(raw)
|
||||
if err != nil || v.Manifest == "" {
|
||||
return nil, fmt.Errorf("the indexer names no manifest of %s", module)
|
||||
}
|
||||
m, err := p.cache.get(p.Read, v.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, heldErr := HeldStreams(p.Call)
|
||||
return Assert(module, v.Manifest, m, held, heldErr), nil
|
||||
}
|
||||
|
||||
// Answer is a tool's arguments read as text.
|
||||
func Answer(args map[string]any, name string) string {
|
||||
switch v := args[name].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case bool:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// User is one bus user and its grants, as the bus module's nats_users answers them: never a password hash.
|
||||
type User struct {
|
||||
User string `json:"user"`
|
||||
Publish []string `json:"publish"`
|
||||
Subscribe []string `json:"subscribe"`
|
||||
AllowResponses bool `json:"allow_responses"`
|
||||
// Hash is the bcrypt hash the vault holds for the user (WithHashes): in memory alone, never in a digest, a compare
|
||||
// or the part's state, and empty for a loaded user.
|
||||
Hash string `json:"-"`
|
||||
}
|
||||
|
||||
// WithHashes gives each user of the shadow list the hash the vault holds for it, when it holds one.
|
||||
func WithHashes(users []User, hashes map[string]HeldHash) []User {
|
||||
for i := range users {
|
||||
users[i].Hash = hashes[users[i].User].Hash
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
// Credential is one user's hash as the list would carry it: whether the vault holds one, and how the user stands in the
|
||||
// shadow list and the loaded list. Never the hash: a bcrypt hash of the same password differs by its salt, so two hashes
|
||||
// say nothing compared.
|
||||
type Credential struct {
|
||||
User string `json:"user"`
|
||||
InShadow bool `json:"in_shadow"`
|
||||
Loaded bool `json:"loaded"`
|
||||
HashHeld bool `json:"hash_held"`
|
||||
Adopted bool `json:"adopted,omitempty"`
|
||||
// Generation is the vault's generation of the account the hash is of; zero for an adopted one.
|
||||
Generation int `json:"generation,omitempty"`
|
||||
// Would is how the list with the vault's hashes would differ from the loaded one for this user: same (a hash held,
|
||||
// loaded), adds (a hash held, not loaded), left-out (in the shadow list, no hash: left out for want of a
|
||||
// credential), loses-login (loaded, no hash held: its login would be lost) or not-composed (a hash held, loaded,
|
||||
// and not in the shadow list).
|
||||
Would string `json:"would"`
|
||||
}
|
||||
|
||||
// The ways a user's line would differ.
|
||||
const (
|
||||
WouldSame = "same"
|
||||
WouldAdd = "adds"
|
||||
WouldLeaveOut = "left-out"
|
||||
WouldLoseLogin = "loses-login"
|
||||
WouldNotComposed = "not-composed"
|
||||
)
|
||||
|
||||
// Credentials is every user's Credential, and the users of each way that differs.
|
||||
type Credentials struct {
|
||||
Users []Credential `json:"users"`
|
||||
HashesHeld int `json:"hashes_held"`
|
||||
LeftOut []string `json:"left_out"`
|
||||
LosesLogin []string `json:"loses_login"`
|
||||
Adds []string `json:"adds"`
|
||||
}
|
||||
|
||||
// CredentialsOf says, of every user of the shadow and the loaded list, whether the vault holds its hash and how its line
|
||||
// would differ. Pure.
|
||||
func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credentials {
|
||||
c := Credentials{Users: []Credential{}, LeftOut: []string{}, LosesLogin: []string{}, Adds: []string{}}
|
||||
rows := map[string]*Credential{}
|
||||
row := func(user string) *Credential {
|
||||
if rows[user] == nil {
|
||||
h, held := hashes[user]
|
||||
rows[user] = &Credential{User: user, HashHeld: held, Adopted: h.Adopted, Generation: h.Generation}
|
||||
}
|
||||
return rows[user]
|
||||
}
|
||||
for _, u := range shadow {
|
||||
row(u.User).InShadow = true
|
||||
}
|
||||
for _, u := range loaded {
|
||||
row(u.User).Loaded = true
|
||||
}
|
||||
for _, user := range sortedKeys(rows) {
|
||||
r := rows[user]
|
||||
switch {
|
||||
case r.HashHeld && r.InShadow && r.Loaded:
|
||||
r.Would = WouldSame
|
||||
case r.HashHeld && r.InShadow:
|
||||
r.Would = WouldAdd
|
||||
c.Adds = append(c.Adds, user)
|
||||
case r.InShadow:
|
||||
r.Would = WouldLeaveOut
|
||||
c.LeftOut = append(c.LeftOut, user)
|
||||
case !r.HashHeld:
|
||||
r.Would = WouldLoseLogin
|
||||
c.LosesLogin = append(c.LosesLogin, user)
|
||||
default:
|
||||
r.Would = WouldNotComposed
|
||||
}
|
||||
if r.HashHeld {
|
||||
c.HashesHeld++
|
||||
}
|
||||
c.Users = append(c.Users, *r)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure.
|
||||
func ComposeUsers(r buscompose.Records) ([]User, error) {
|
||||
principals, err := buscompose.Users(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]User, 0, len(principals))
|
||||
for _, p := range principals {
|
||||
perms, err := buscompose.PermissionsFor(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", p.Username(), err)
|
||||
}
|
||||
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
|
||||
AllowResponses: perms.AllowResponses})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func set(in []string) []string {
|
||||
out := slices.Clone(in)
|
||||
sort.Strings(out)
|
||||
out = slices.Compact(out)
|
||||
if out == nil {
|
||||
out = []string{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Digest is a list's digest, `sha256:<hex>`, over its users and grants in order.
|
||||
func Digest(users []User) string {
|
||||
raw, _ := json.Marshal(users)
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// LoadedUsers is the list the bus loaded, as its module's nats_users reads it from the file: every user by name, then
|
||||
// each one's grants in full.
|
||||
func LoadedUsers(c Caller) ([]User, error) {
|
||||
raw, err := c.Call(CallUsers, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", CallUsers, err)
|
||||
}
|
||||
var names struct {
|
||||
Users []struct {
|
||||
User string `json:"user"`
|
||||
} `json:"users"`
|
||||
}
|
||||
if err := json.Unmarshal(unwrap(raw), &names); err != nil {
|
||||
return nil, fmt.Errorf("%s answered no list of users: %w", CallUsers, err)
|
||||
}
|
||||
full := map[string]User{}
|
||||
ask := func(part string) error {
|
||||
raw, err := c.Call(CallUsers, map[string]any{"user": part})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s: %w", CallUsers, part, err)
|
||||
}
|
||||
var got struct {
|
||||
Users []User `json:"users"`
|
||||
}
|
||||
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
|
||||
return fmt.Errorf("%s %s answered no grants: %w", CallUsers, part, err)
|
||||
}
|
||||
for _, u := range got.Users {
|
||||
u.Publish, u.Subscribe = set(u.Publish), set(u.Subscribe)
|
||||
full[u.User] = u
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Every name with a dot in one call; each name without one (the controller) alone.
|
||||
if err := ask("."); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]User, 0, len(names.Users))
|
||||
for _, n := range names.Users {
|
||||
if _, ok := full[n.User]; !ok {
|
||||
if err := ask(n.User); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
u, ok := full[n.User]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s names %s and gives no grants for it", CallUsers, n.User)
|
||||
}
|
||||
out = append(out, u)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unwrap takes a tool's answer out of an MCP content wrapper, when a runtime hands one.
|
||||
func unwrap(raw json.RawMessage) []byte {
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) == 1 && wrapped.Content[0].Text != "" {
|
||||
return []byte(wrapped.Content[0].Text)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// UserDiff is how one user's grants differ: what the shadow list grants that the loaded list does not, and back.
|
||||
type UserDiff struct {
|
||||
User string `json:"user"`
|
||||
PublishAdded []string `json:"publish_added,omitempty"`
|
||||
PublishDropped []string `json:"publish_dropped,omitempty"`
|
||||
SubscribeAdded []string `json:"subscribe_added,omitempty"`
|
||||
SubscribeDropped []string `json:"subscribe_dropped,omitempty"`
|
||||
Responses string `json:"responses,omitempty"`
|
||||
}
|
||||
|
||||
// Comparison is the shadow list against the loaded list.
|
||||
type Comparison struct {
|
||||
Shadow int `json:"shadow"`
|
||||
Loaded int `json:"loaded"`
|
||||
// Added are users in the shadow list the loaded list lacks (one left out for want of a credential among them);
|
||||
// Dropped are loaded users the shadow list lacks; Gone are those of Dropped whose machine or module placement no
|
||||
// longer names, which both lists agree are gone.
|
||||
Added []string `json:"added"`
|
||||
Dropped []string `json:"dropped"`
|
||||
Gone []string `json:"gone,omitempty"`
|
||||
Differ []UserDiff `json:"differ"`
|
||||
Same int `json:"same"`
|
||||
// Unread says the loaded list could not be read, and why: nothing is compared then.
|
||||
Unread string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// Equal says the two lists agree in every user, but those both agree are gone.
|
||||
func (c Comparison) Equal() bool {
|
||||
return c.Unread == "" && len(c.Added) == 0 && len(c.Differ) == 0 && len(c.Dropped) == len(c.Gone)
|
||||
}
|
||||
|
||||
// Differing names every user that keeps the lists apart: added, dropped and not gone, or granted otherwise.
|
||||
func (c Comparison) Differing() []string {
|
||||
var out []string
|
||||
out = append(out, c.Added...)
|
||||
for _, d := range c.Dropped {
|
||||
if !slices.Contains(c.Gone, d) {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
for _, d := range c.Differ {
|
||||
out = append(out, d.User)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Compare compares the shadow list with the loaded one. placed says whether placement still names a loaded user's
|
||||
// machine and module; nil names every one. Pure.
|
||||
func Compare(shadow, loaded []User, placed func(user string) bool) Comparison {
|
||||
c := Comparison{Shadow: len(shadow), Loaded: len(loaded), Added: []string{}, Dropped: []string{}, Differ: []UserDiff{}}
|
||||
have := map[string]User{}
|
||||
for _, u := range loaded {
|
||||
have[u.User] = u
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, s := range shadow {
|
||||
want[s.User] = true
|
||||
l, ok := have[s.User]
|
||||
if !ok {
|
||||
c.Added = append(c.Added, s.User)
|
||||
continue
|
||||
}
|
||||
d := UserDiff{User: s.User, PublishAdded: minus(s.Publish, l.Publish), PublishDropped: minus(l.Publish, s.Publish),
|
||||
SubscribeAdded: minus(s.Subscribe, l.Subscribe), SubscribeDropped: minus(l.Subscribe, s.Subscribe)}
|
||||
if s.AllowResponses != l.AllowResponses {
|
||||
d.Responses = fmt.Sprintf("shadow %v, loaded %v", s.AllowResponses, l.AllowResponses)
|
||||
}
|
||||
if len(d.PublishAdded)+len(d.PublishDropped)+len(d.SubscribeAdded)+len(d.SubscribeDropped) == 0 && d.Responses == "" {
|
||||
c.Same++
|
||||
continue
|
||||
}
|
||||
c.Differ = append(c.Differ, d)
|
||||
}
|
||||
for _, l := range loaded {
|
||||
if !want[l.User] {
|
||||
c.Dropped = append(c.Dropped, l.User)
|
||||
if placed != nil && !placed(l.User) {
|
||||
c.Gone = append(c.Gone, l.User)
|
||||
}
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func minus(a, b []string) []string {
|
||||
var out []string
|
||||
for _, x := range a {
|
||||
if !slices.Contains(b, x) {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Placed says whether placement still names a user's machine and module: a machine's users (`node.<n>`,
|
||||
// `provisioner.<n>`) while the machine is seen, a module's or process's (`<n>.<module>[.<process>]`) while the module is
|
||||
// assigned there. The controller, enrolment users and persons are never judged gone from here.
|
||||
func Placed(in Inputs) func(user string) bool {
|
||||
return func(user string) bool {
|
||||
parts := strings.Split(user, ".")
|
||||
switch {
|
||||
case len(parts) < 2:
|
||||
return true
|
||||
case parts[0] == "node" || parts[0] == "provisioner":
|
||||
return slices.Contains(in.Nodes, parts[1])
|
||||
case parts[0] == "person" || parts[0] == "enrol":
|
||||
return true
|
||||
}
|
||||
return slices.Contains(in.Nodes, parts[0]) && slices.Contains(in.Assigned[parts[0]], parts[1])
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user