Compare commits

..
82 changed files with 444 additions and 7805 deletions
@@ -182,42 +182,6 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
t.Errorf("without %s the assigner started", missing)
}
}
// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
// ADR 0201) a pass opens it, so a put is taken further without a restart.
was := PutsReopenEvery
PutsReopenEvery = 0
defer func() { PutsReopenEvery = was }()
b := startBed(t, delivery.AssignerPutsBucket)
done := b.start(t)
select {
case err := <-done:
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
case <-time.After(2 * time.Second):
}
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PutKey("n1", "web")
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
raw, _ := delivery.EncodePut(asked)
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
t.Fatal(err)
}
if e, err := puts.Get(b.ctx, pkey); err == nil {
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
break
}
}
if time.Now().After(deadline) {
t.Fatal("the put was not taken further once its bucket was made")
}
}
}
// putJSON writes a value into one of the bed's buckets.
@@ -16,7 +16,6 @@ import (
"os/signal"
"strings"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -86,11 +85,11 @@ var keptStates = []announce.State{
// buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
// without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) {
out := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
@@ -99,7 +98,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
continue
}
if err != nil {
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
}
out[name] = kv
}
@@ -214,9 +213,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return
}
if e == nil {
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
settled <- struct{}{}
}
settled <- struct{}{}
continue
}
var value []byte
@@ -230,39 +227,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}()
return nil
}
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
// stops the puts, never the wants.
followed := watched
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
logger.Printf("%v: no put is taken further until the assigner starts again", err)
delete(kvs, delivery.AssignerPutsBucket)
} else {
followed = append(append([]string{}, watched...), delivery.AssignerPutsBucket)
}
}
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
if err != nil {
return
}
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
for _, name := range watched {
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket {
@@ -275,7 +248,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
for range len(watched) + 1 {
for range len(followed) + 1 {
select {
case <-ctx.Done():
return nil
@@ -298,15 +271,11 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return err
case <-settled:
case <-dirty:
reopenPuts(p)
p.pass(ctx)
}
}
}
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
var PutsReopenEvery = time.Minute
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
@@ -5,18 +5,15 @@ import (
"errors"
"fmt"
"io"
"log"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -73,36 +70,10 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)):
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
}
}
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
// keep the first pass from being made: the wants are written all the same.
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
ckey, _ := delivery.ComposedNodeKey("ace")
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
if err != nil {
t.Fatal(err)
}
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
t.Errorf("the first pass wrote %s", key)
}
case <-ctx.Done():
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
}
}
// 1297/B2a: without a credential the program does not start, and says why.
@@ -126,47 +97,3 @@ func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue
}
return &memKV{values: map[string][]byte{}}, nil
}
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
// closed; a put on it is said on wrote.
type watchKV struct {
*memKV
entries []kvEntry
closed bool
wrote chan string
}
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
if w.closed {
close(ch)
return keyWatcher{ch}, nil
}
for _, e := range w.entries {
ch <- e
}
ch <- nil
return keyWatcher{ch}, nil
}
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
if w.wrote != nil {
w.wrote <- key
}
return 1, nil
}
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
func (k keyWatcher) Stop() error { return nil }
type kvEntry struct {
jetstream.KeyValueEntry
key string
value []byte
}
func (e kvEntry) Key() string { return e.key }
func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
@@ -3,8 +3,6 @@ package main
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
@@ -37,9 +35,6 @@ type passer struct {
neither map[string]neitherOne
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
acts *putActs
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
}
func (p *passer) clock() time.Time {
@@ -99,7 +94,6 @@ func (p *passer) wants(ctx context.Context) {
}
wanted := p.w.wantsOf()
puts := p.putsByModule()
p.staleAsks(ctx, trunk)
names := map[string]bool{}
for name := range trunk {
names[name] = true
@@ -154,7 +148,6 @@ func (p *passer) wants(ctx context.Context) {
}
out := assign.Wants(m, nodes, wanted[name], last)
gatherNeither(neither, name, out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
if first {
startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
@@ -193,42 +186,6 @@ func (p *passer) wants(ctx context.Context) {
}
}
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
if p.noWant == nil {
p.noWant = map[string]string{}
}
assigned := map[string]bool{}
for _, node := range nodes {
assigned[node] = true
key := node + "." + module
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
reason := ""
if h, held := out.Rules[node]; held && (!written || w.Removed) {
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
last, said := p.noWant[key]
switch {
case reason != "" && reason != last:
p.logf("%s on %s gets no want: %s", module, node, reason)
p.noWant[key] = reason
case reason == "" && said && written && !w.Removed:
p.logf("%s on %s has a want", module, node)
delete(p.noWant, key)
}
}
for key := range p.noWant {
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
delete(p.noWant, key) // no longer assigned there: its removed want says so
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct {
node, module string
@@ -406,92 +363,6 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
}
p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask)
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
// set back from it (issue 1906, note 28).
for key := range p.w.read(delivery.AssignerAsksBucket) {
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, delivery.AssignerAsksBucket, key)
}
}
}
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
// nothing taken away.
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
k, err := delivery.ParseAnyAskKey(key)
if err != nil {
return false
}
switch k.Kind {
case delivery.AskKindHold, delivery.AskKindRestore:
m, on := trunk[k.Module]
return !on || (m.FP2 != "" && m.FP2 != k.FP)
case delivery.AskKindRecord:
m, on := trunk[k.Module]
if !on {
return true
}
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
// trunk's, its want's, its have's), and only when every one of them is read.
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
d, ok := p.w.composed(k.Node, k.Module, fp)
if !ok || d.Digest == k.Composed {
return false
}
}
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
}
return false
}
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
// names, each once.
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
var out []string
add := func(fp string) {
if fp != "" && !slices.Contains(out, fp) {
out = append(out, fp)
}
}
add(trunkFP)
if key, err := delivery.WantKey(node, module); err == nil {
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
add(w.FP)
}
}
if key, err := delivery.HaveKey(node, module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
add(h.FP)
}
}
return out
}
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
// read at all (no module), nothing is taken away.
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
if len(trunk) == 0 {
return
}
for key := range p.w.read(delivery.AssignerAsksBucket) {
if p.staleAsk(key, trunk) {
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
p.remove(ctx, delivery.AssignerAsksBucket, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
if k, err := delivery.ParseAnyAskKey(key); err == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore) {
p.clearHeld(k.Module, k.FP)
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
}
}
}
}
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
@@ -381,31 +380,6 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
t.Errorf("gathered = %+v", got)
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
for i, c := range []struct {
before map[string]delivery.Want
out assign.Outcome
line string // the line said in this pass, "" for none
}{
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
{nil, notDone, ""},
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
if got := strings.Join(logged, "\n"); got != c.line {
t.Errorf("pass %d said %q, want %q", i, got, c.line)
}
}
}
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
@@ -496,190 +470,3 @@ func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
}
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
trunkOf := func(fp string) map[string]delivery.Module {
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
}
type ask struct{ name, key string }
holdOld, _ := delivery.HoldAskKey("web", fpOld)
holdNew, _ := delivery.HoldAskKey("web", fpNew)
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
for _, c := range []struct {
name string
trunk map[string]delivery.Module
gone []string // the keys taken away
noDigit bool // the controller's composition is not read
heldAt string // the node's want is at this fingerprint, whose composition is digOld
}{
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
{"the trunk is not read at all", nil, nil, false, ""},
} {
t.Run(c.name, func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
for _, key := range all {
k, _ := delivery.ParseAnyAskKey(key)
var a delivery.AssignerAsk
switch k.Kind {
case delivery.AskKindHold:
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
case delivery.AskKindRestore:
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
case delivery.AskKindRecord:
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
default:
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
}
v, err := delivery.EncodeAssignerAsk(key, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[key] = v
p.w.put(delivery.AssignerAsksBucket, key, v)
}
if !c.noDigit {
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
p.w.put(delivery.ControllerComposedBucket, ckey, v)
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
if c.heldAt != "" {
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
}
p.w.put(delivery.ControllerComposedBucket, okey, old)
}
if c.heldAt != "" {
wkey, _ := delivery.WantKey("novox", "web")
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
p.w.put(delivery.AssignerWantsBucket, wkey, w)
}
heldOld, _ := delivery.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
p.staleAsks(context.Background(), c.trunk)
// the held record and condition of a fingerprint whose hold ask went are taken away with it
heldOld, _ = delivery.HeldKey("web", fpOld)
_, heldThere := wantsKV.values[heldOld]
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
})
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
}
for _, key := range all {
_, there := asksKV.values[key]
want := !slices.Contains(c.gone, key)
if there != want {
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
}
}
if slices.Contains(c.gone, holdOld) {
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
p.now = func() time.Time { return time.Now().Add(time.Hour) }
p.holdAsk(context.Background(), "web", fpOld, nil)
p.heldCondition(context.Background(), "web", fpOld, nil, false)
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
}
}
})
}
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
asksKV.values[key] = []byte("x")
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
}
p.lift(context.Background(), "web", fpOld)
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
if _, there := asksKV.values[key]; there == gone {
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
}
}
})
}
func mustKey(key string, err error) string {
if err != nil {
panic(err)
}
return key
}
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
// yes the want names the revised digest.
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
if cerr != nil {
t.Fatal(cerr)
}
put(delivery.ControllerComposedBucket, ckey, cd)
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
put(delivery.ControllerComposedBucket, tkey, tcd)
akey, _ := delivery.RecordAskKey("g14", "web", revised)
for i := 0; i < 2; i++ {
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
}
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil {
t.Fatal(err)
}
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
p.wants(context.Background())
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != revised {
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
}
if _, there := asksKV.values[akey]; !there {
t.Error("the answered record ask was taken away")
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -6,8 +6,6 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -226,15 +226,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
before, had := wants[node]
var next delivery.Want
p, put := m.Puts[node]
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: delivery.PutHolding}, true
}
if put {
if p, put := m.Puts[node]; put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
@@ -333,18 +325,8 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != delivery.PhaseRemoved && !held:
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == delivery.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
if !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
continue
}
@@ -380,11 +362,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out
}
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
}
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
// moving). A failed health while it
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
@@ -397,15 +397,10 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
} {
p := put
p.Step = c.step
m.Puts = map[string]delivery.Put{"ace": p}
if c.step == "" {
m.Puts = nil
}
out := Wants(m, n, c.wanted, nil)
got, wrote := out.Wants["ace"]
switch {
@@ -867,33 +862,6 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
}
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
// runs the module is on neither list, its removed want left as it is.
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
for _, c := range []struct {
name string
before map[string]delivery.Want
want map[string]delivery.Want
}{
{"no key: nothing written", nil, map[string]delivery.Want{}},
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
map[string]delivery.Want{}},
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
} {
out := Wants(m, n, c.before, nil)
if len(out.Wants) != len(c.want) {
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
continue
}
for node, w := range c.want {
if !delivery.SameWant(normal(out.Wants[node]), w) {
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
}
}
}
m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
@@ -1,178 +0,0 @@
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.StuckTool,
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
{Name: build.AskAgainTool,
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := askAgain(key, value); err != nil {
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
}
return map[string]any{"key": key, "asked": true,
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
}},
{Name: build.SeatQueueTool,
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
Run: func(args map[string]any) (any, error) {
language := build.LanguageOf(str(args, "language"))
if language == "" {
language = build.LanguageOf(str(args, "seat"))
}
if language == "" {
language = delivery.AskGo
}
limit, _ := args["limit"].(float64)
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
return queue(key, value)
}},
}
}
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
return func(key string, value []byte) (json.RawMessage, error) {
answered := make(chan json.RawMessage, 1)
var rev uint64
var mu sync.Mutex
if err := s.Watch(key, func(c stdio.StateChange) error {
mu.Lock()
mine := rev
mu.Unlock()
if c.Op != "put" || mine == 0 || c.Revision <= mine {
return nil
}
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
select {
case answered <- c.Value:
default:
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
return e.Value, nil
}
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case v := <-answered:
return v, nil
case <-wait.Done():
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
}
}
}
func read(s stdio.KeptState) build.StateRead {
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
e, err := s.Get(key)
if err != nil || e == nil {
return nil, time.Time{}, err
}
return e.Value, time.Time{}, nil
}, Written: func(key string) time.Time {
e, err := s.Get(key)
if err != nil || e == nil {
return time.Time{}
}
return written(s, key, e.Revision)
}}
}
// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
h, err := s.History(key)
if err != nil {
return time.Time{}
}
for _, r := range h {
if r.Revision == revision {
return r.At
}
}
return time.Time{}
}
func main() {
steps := map[string]build.StateRead{}
for _, bm := range build.StuckBuildModules {
steps[bm] = read(stdio.State(bm + ".steps"))
}
reasks := stdio.State(build.ReAsksState)
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
put := func(key string, value []byte) error {
_, err := reasks.Put(key, json.RawMessage(value))
return err
}
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
}
}
@@ -4,303 +4,73 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
// be answered (issue 1945). Never polled.
// (issue 1910): read once, at the take-over of a key at start, never polled.
type heldOnTheBus struct{ js jetstream.JetStream }
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
// the ask's subject whose Nats-Msg-Id is the ask's id.
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
seen, err := h.seen(ctx, a, false)
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return false, err
}
return seen.Step || seen.OnStream, nil
}
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
seen, err := h.seen(ctx, a, true)
pattern, err := delivery.AskStepWatch(a.ID)
if err != nil {
return "", err
}
return seen.Why(), nil
}
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
const (
maxOnSubject = 5000
maxSteps = 200
)
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
// the ask is on its stream.
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return seen, err
return false, err
}
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
pattern, err := delivery.AskStepWatch(a.ID)
if err != nil {
return seen, err
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
keys, err := steps.ListKeysFiltered(ctx, pattern)
if err != nil {
return seen, err
return false, err
}
for range keys.Keys() {
keys.Stop()
seen.Step = true // a node took it
return seen, nil
}
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
seen.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
return seen, nil
return true, nil // a node took it
}
subject, err := delivery.AskSubject(a.Language, a.Kind)
if err != nil {
return seen, err
}
var ahead []build.Queued
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
if q.Ask == a.ID {
seen.OnStream, seen.PutAt = true, q.PutAt
return false
}
ahead = append(ahead, q)
return true
})
if err != nil || !seen.OnStream || !later {
return seen, err
}
taken, err := stepKeys(ctx, steps)
if err != nil {
return seen, err
}
for i := range ahead {
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
}
seen.Ahead = ahead
since := seen.At
if since.IsZero() {
since = seen.PutAt
}
for _, q := range ahead {
if q.At.Before(since) {
since = q.At
}
}
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
return seen, nil
}
// stream is the build seat's stream of language, its name and the subject of kind on it.
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
subject, err := delivery.AskSubject(language, kind)
if err != nil {
return nil, "", "", err
return false, err
}
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
// messages by sequence and nothing wider (issue 1910).
name, err := delivery.AskStream(language)
name, err := delivery.AskStream(a.Language)
if err != nil {
return nil, "", "", err
return false, err
}
stream, err := h.js.Stream(ctx, name)
return stream, name, subject, err
}
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
// none consumed.
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
for seq, n := uint64(1), 0; ; n++ {
if n >= maxOnSubject {
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
}
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
if errors.Is(err, jetstream.ErrMsgNotFound) {
return nil
}
if err != nil {
return err
}
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
q.Kind = kind
}
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
q.At = q.PutAt
}
if !each(q) {
return nil
}
seq = m.Sequence + 1
}
}
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
// ask id and is left out).
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
all, err := steps.ListKeys(ctx)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
return map[string][]string{}, nil
}
return nil, err
}
defer all.Stop()
out := map[string][]string{}
for k := range all.Keys() {
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
out[id] = append(out[id], k)
}
}
return out, nil
}
// stepState is the state of the first of keys a node wrote, "" for none.
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
for _, k := range keys {
e, err := steps.Get(ctx, k)
if err != nil {
continue
}
if s, err := delivery.DecodeStep(e.Value()); err == nil {
return s.State
}
}
return ""
}
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
// (mesh-sdk buildstep).
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
var ids []string
for id := range taken {
if build.AskedAt(id).After(since) {
ids = append(ids, id)
}
}
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
read := 0
for _, id := range ids {
for _, k := range taken[id] {
if read >= maxSteps {
return nil
}
read++
e, err := steps.Get(ctx, k)
if err != nil {
continue
}
s, err := delivery.DecodeStep(e.Value())
if err != nil {
continue
}
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
if !ok || !build.Proves(stepKind, kind) {
continue
}
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
}
}
return nil
}
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
if errors.Is(err, jetstream.ErrStreamNotFound) {
return nil
return false, nil
}
if err != nil {
return err
}
var seq uint64
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
if q.Ask == a.ID {
seq = q.Seq
return false
}
return true
}); err != nil || seq == 0 {
return err
}
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
}
return nil
}
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
q := build.SeatQueue{Language: language, Note: build.QueueNote}
module, err := delivery.BuildModuleOf(language)
if err != nil {
return q, err
}
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
q.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
q.Note = name + " does not exist"
return q, nil
}
if err != nil {
return q, err
return false, err
}
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
// instead, each a direct read that consumes nothing.
info, err := stream.Info(ctx)
if err != nil {
return q, err
return false, err
}
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
taken, err := stepKeys(ctx, steps)
if err != nil {
return q, err
}
for _, kind := range delivery.AskKinds {
subject, _ := delivery.AskSubject(language, kind)
var ahead []build.Queued
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
ahead = append(ahead, m)
return len(ahead) < limit
}); err != nil {
return q, err
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
m, err := stream.GetMsg(ctx, seq)
if errors.Is(err, jetstream.ErrMsgNotFound) {
continue // taken and acknowledged, or removed
}
if len(ahead) == 0 {
continue
if err != nil {
return false, err
}
for i := range ahead {
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
return true, nil
}
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
}
return q, nil
return false, nil
}
@@ -24,8 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
@@ -824,34 +822,6 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
}
}
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
held := heldOnTheBus{b.js}
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
t.Fatal(err)
}
queued := func() []build.QueuedState {
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
}
return q.Kinds[0].Messages
}
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
}
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
}
if err := held.Drop(b.ctx, stale); err != nil {
t.Fatal(err)
}
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
}
b.stop()
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
b.fake.open()
@@ -13,6 +13,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"io"
"log"
"maps"
@@ -24,8 +25,6 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -125,20 +124,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
builds, err := bucket(delivery.BuilderBuildsBucket)
if err != nil {
return err
}
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
modules, err := bucket(delivery.ModulesBucket)
if err != nil {
return err
}
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil {
return err
}
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
bound := defaultStepBound
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
if bound, err = time.ParseDuration(v); err != nil {
@@ -161,36 +169,44 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
CheckAfter: bound,
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js},
Toolchains: tools,
Manifests: manifestReader{manifests},
Logf: logger.Printf,
Stored: storedVia(calls.Call),
}
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
errs := make(chan error, 6)
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
trunks.start()
b.Trunk = trunks.head
b.Said = func(e build.AskedAgain) {
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
logger.Printf("%v", err)
// revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said.
// The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a
// revision heard after the candidate that names it builds that candidate then.
trunks := &sourceHeads{}
var sw jetstream.KeyWatcher
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err)
}
if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil {
return err
}
b.Trunk = trunks.head
}
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
// is asked since R4g, ADR 0524 §7).
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
case kv == nil:
switch kv, err := js.KeyValue(ctx, instancesBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
default:
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
b.Answered = says.Answered
@@ -270,36 +286,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
b.ForgetCandidate(e.Key())
}
}
if candidates != nil {
cw, err := settle(ctx, candidates, eachCandidate)
if err != nil {
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
} else {
go func() { errs <- follow(ctx, cw, eachCandidate) }()
}
}
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
} else {
go func() { errs <- follow(ctx, rw, eachReAsk) }()
}
}
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
// builder alone may read the stream, so it answers there.
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
if qw, err := settle(ctx, kv, eachRead); err != nil {
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
} else {
go func() { errs <- follow(ctx, qw, eachRead) }()
}
cw, err := settle(ctx, candidates, eachCandidate)
if err != nil {
return err
}
logger.Printf("started on %s", cred.Node)
errs := make(chan error, 3)
go func() { errs <- follow(ctx, w, each) }()
go func() { errs <- follow(ctx, cw, eachCandidate) }()
if sw != nil {
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if trunks.put(e) {
b.TrunkMoved(ctx)
}
})
}()
}
select {
case <-ctx.Done():
b.Wait()
@@ -1,78 +0,0 @@
package main
import (
"context"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
// and opened again at the next event that needs it where there is one.
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
// and a candidate's preview waits, saying why.
type trunkSource struct {
ctx context.Context
js bucketOpener
heads *sourceHeads
moved func(context.Context)
errs chan<- error
logf func(string, ...any)
mu sync.Mutex
opened bool
}
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
func (t *trunkSource) ensure() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.opened {
return nil
}
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
if err != nil {
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
}
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
}
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
if err != nil {
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
}
t.opened = true
go func() {
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
if t.heads.put(e) && t.moved != nil {
t.moved(t.ctx)
}
})
}()
return nil
}
// start opens the source at start; one that cannot be is said, and the builder goes on.
func (t *trunkSource) start() {
if err := t.ensure(); err != nil {
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
}
}
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
if err := t.ensure(); err != nil {
return "", err
}
return t.heads.head(ctx, rev, repository)
}
@@ -1,49 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
type queueReader interface {
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
}
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
// instance's) or deleted is left.
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
a, err := build.DecodeQueueAsk(e.Value())
if err != nil {
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
return
}
if !a.Answered.IsZero() {
return
}
read, cancel := context.WithTimeout(ctx, 20*time.Second)
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
cancel()
if err != nil {
a.Error = err.Error()
}
a.Queue, a.Answered = &q, time.Now().UTC()
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
}
}
@@ -1,75 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// askAgainer is the part of the builder a reasks value asks through.
type askAgainer interface {
AskAgain(key, why string) []string
}
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
// answered already (this builder's own write) or deleted is left.
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
r, err := build.DecodeReAsk(e.Value())
if err != nil {
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
return
}
if r.Answer != "" {
return
}
asked := b.AskAgain(e.Key(), r.Why)
if len(asked) == 0 {
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
} else {
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
}
r.Answered = time.Now().UTC()
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
data, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
id := "asked-again-" + e.New
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
}
return nil
}
@@ -193,9 +193,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
@@ -3,19 +3,11 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -97,61 +89,6 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
@@ -4,13 +4,11 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"log"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -64,14 +62,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
return nil
return err
}
w.watched[module] = true
go func() {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,337 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
const DefaultAskings = 3
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
const AskedAgainEvent = "build.asked-again"
// AskedAgain is the body of AskedAgainEvent.
type AskedAgain struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp"`
BuildModule string `json:"build_module"`
Language string `json:"language"`
Old string `json:"old"`
New string `json:"new"`
Asking int `json:"asking"`
Askings int `json:"askings"`
Why string `json:"why"`
At time.Time `json:"at"`
}
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
// five hours, each counted as waiting, so the ask was never asked again).
type AskSeen struct {
Stream string
// Kind is the ask's kind, At when it was asked (its id's time).
Kind string
At time.Time
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
Step bool
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
OnStream bool
PutAt time.Time
// Ahead are the messages of its subject before it on the stream.
Ahead []Queued
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
Last *Taken
}
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
type Queued struct {
Seq uint64 `json:"seq"`
Ask string `json:"ask"`
Kind string `json:"kind"`
At time.Time `json:"asked"`
PutAt time.Time `json:"put"`
Step string `json:"step,omitempty"`
}
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
type Taken struct {
Ask string `json:"ask"`
Kind string `json:"kind"`
At time.Time `json:"asked"`
Node string `json:"node"`
Started time.Time `json:"started"`
}
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
// its worker had none of waiting's kind left to give.
func Proves(taken, waiting string) bool {
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
return t >= 0 && w >= 0 && t >= w
}
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
// its age, and is not counted.
func (s AskSeen) Waits(q Queued) bool {
return q.Step == "" && !s.passed(q.At)
}
func (s AskSeen) passed(at time.Time) bool {
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
}
// Waiting is how many messages ahead still wait to be delivered (Waits).
func (s AskSeen) Waiting() int {
n := 0
for _, q := range s.Ahead {
if s.Waits(q) {
n++
}
}
return n
}
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
// ADR 0525 §9).
func (s AskSeen) Why() string {
switch {
case s.Step:
return ""
case !s.OnStream:
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
case s.Waiting() > 0 || !s.passed(s.at()):
return ""
default:
behind := ""
if n := len(s.Ahead); n > 0 {
behind = fmt.Sprintf(" behind %d given up on or taken", n)
}
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
}
}
func (s AskSeen) at() time.Time {
if s.At.IsZero() {
return s.PutAt
}
return s.At
}
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
func AskedAt(id string) time.Time {
if len(id) <= 6 {
return time.Time{}
}
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
if err != nil || ms <= 0 {
return time.Time{}
}
return time.UnixMilli(ms)
}
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
type Dropper interface {
Drop(ctx context.Context, a delivery.StepAsk) error
}
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
d, ok := b.Gone.(Dropper)
if !ok {
return
}
prior := ask
prior.ID = old
if err := d.Drop(ctx, prior); err != nil {
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
}
}
func (b *Builder) askings() int {
if b.Askings > 0 {
return b.Askings
}
return DefaultAskings
}
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
func askingOf(step, buildModule string) int {
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
if m[1] == buildModule {
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
return n
}
}
}
return 1
}
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
// languages whose lane waits, none when no lane of this builder waits on the key.
func (b *Builder) AskAgain(key, why string) []string {
b.init()
b.mu.Lock()
defer b.mu.Unlock()
var asked []string
for language, c := range b.waiting[key] {
select {
case c <- why:
default: // one is already asked
}
asked = append(asked, language)
}
return asked
}
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
func (b *Builder) Waiting() map[string][]string {
b.init()
b.mu.Lock()
defer b.mu.Unlock()
out := map[string][]string{}
for key, ls := range b.waiting {
for l := range ls {
out[key] = append(out[key], l)
}
}
return out
}
func (b *Builder) listen(key, language string) chan string {
b.init()
c := make(chan string, 1)
b.mu.Lock()
if b.waiting[key] == nil {
b.waiting[key] = map[string]chan string{}
}
b.waiting[key][language] = c
b.mu.Unlock()
return c
}
func (b *Builder) unlisten(key, language string, c chan string) {
b.mu.Lock()
if b.waiting[key][language] == c {
delete(b.waiting[key], language)
if len(b.waiting[key]) == 0 {
delete(b.waiting, key)
}
}
b.mu.Unlock()
}
var (
errCheck = errors.New("the check bound passed")
errAgain = errors.New("asked again")
)
type byHand struct{ why string }
func (h byHand) Error() string { return "asked again by hand: " + h.why }
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
// asked it again (AskAgain); it answers the step and the ask that has it.
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
key, _ := delivery.BuilderBuildKey(module, fp)
again := b.listen(key, ask.Language)
defer b.unlisten(key, ask.Language, again)
asking := askingOf(asked.step, l.module)
for {
step, why, err := b.waitOnce(ctx, ask, again)
switch {
case errors.Is(err, errCheck):
continue
case errors.Is(err, errAgain):
if asking >= b.askings() {
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
}
asking++
old := ask.ID
ask.ID, ask.At = buildstep.NewID(), time.Now()
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
}
if err := b.Asks.Put(ctx, ask); err != nil {
return step, ask, err
}
b.drop(ctx, ask, old)
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
if b.Said != nil {
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
}
default:
return step, ask, err
}
}
}
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
hand, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
go func() {
select {
case why := <-again:
cancel(byHand{why})
case <-hand.Done():
}
}()
wait := hand
if b.Gone != nil && b.CheckAfter > 0 {
var stop context.CancelFunc
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
defer stop()
}
step, err := b.Asks.Wait(wait, ask)
if err == nil || ctx.Err() != nil {
return step, "", err
}
var h byHand
if errors.As(context.Cause(hand), &h) {
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
}
if errors.Is(context.Cause(wait), errCheck) {
why, gerr := b.Gone.Gone(ctx, ask)
if gerr != nil {
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
return step, "", errCheck
}
if why == "" {
return step, "", errCheck
}
return step, why, errAgain
}
return step, "", err
}
@@ -1,115 +0,0 @@
package build
import (
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
// there, as builder_ask_again does through reasks.
const (
SeatQueueTool = "builder_seat_queue"
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
QueuesState = "queues"
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
DefaultQueueLimit = 50
MaxQueueLimit = 500
)
// QueueNote says what the read cannot show.
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
type QueueAsk struct {
V int `json:"v"`
Limit int `json:"limit"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Answered time.Time `json:"answered,omitzero"`
Error string `json:"error,omitempty"`
Queue *SeatQueue `json:"queue,omitempty"`
}
// SeatQueue is what the builder read of one build seat's stream.
type SeatQueue struct {
Language string `json:"language"`
Stream string `json:"stream"`
Messages uint64 `json:"messages"`
FirstSeq uint64 `json:"first_seq"`
LastSeq uint64 `json:"last_seq"`
Kinds []KindQueue `json:"kinds"`
Note string `json:"note"`
}
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
// last a build node took of the kind (or of one pulled after it), and each message.
type KindQueue struct {
Kind string `json:"kind"`
Read int `json:"read"`
Waiting int `json:"waiting"`
GivenUp int `json:"given_up_or_taken"`
Last *Taken `json:"last_taken,omitempty"`
Messages []QueuedState `json:"messages"`
}
// QueuedState is one message as the tool shows it.
type QueuedState struct {
Queued
AgeSeconds int64 `json:"age_s"`
Waits bool `json:"waits"`
}
// NewKindQueue is seen's messages ahead as one kind's queue at now.
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
for _, q := range seen.Ahead {
w := seen.Waits(q)
if w {
k.Waiting++
} else {
k.GivenUp++
}
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
}
return k
}
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
if _, err := delivery.AskStream(language); err != nil {
return "", nil, fmt.Errorf("language: %w", err)
}
if limit <= 0 {
limit = DefaultQueueLimit
}
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
return language, data, err
}
// DecodeQueueAsk reads a queues value.
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
var a QueueAsk
if err := json.Unmarshal(data, &a); err != nil {
return a, err
}
if a.V != 1 {
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
}
return a, nil
}
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
func LanguageOf(arg string) string {
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
}
@@ -56,18 +56,6 @@ type Builder struct {
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
Gone interface {
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
}
CheckAfter time.Duration
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
// with why; DefaultAskings when 0.
Askings int
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
Said func(AskedAgain)
once sync.Once
sem chan struct{}
@@ -87,8 +75,6 @@ type Builder struct {
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
held map[string]bool
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
waiting map[string]map[string]chan string
}
type planned struct {
@@ -105,7 +91,7 @@ func (b *Builder) init() {
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.published, b.held = map[string]bool{}, map[string]bool{}
})
}
@@ -687,14 +673,12 @@ type askNotes struct {
b *Builder
cl *Claim
resumed map[string]string
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
step string
}
func (b *Builder) notes(cl *Claim) *askNotes {
n := &askNotes{b: b, cl: cl}
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
n.resumed = maps.Clone(cl.Build.Asks)
}
return n
}
@@ -792,7 +776,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
return
}
}
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
step, err := b.Asks.Wait(ctx, ask)
if err != nil {
l.err = fmt.Errorf("%s: %w", l.module, err)
return
@@ -6,10 +6,8 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -618,286 +616,27 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
}
}
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
// drop from their stream (Dropper).
type goneAsks struct {
mu sync.Mutex
n int
why func(a delivery.StepAsk) string
dropped []string
}
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
g.mu.Lock()
defer g.mu.Unlock()
g.dropped = append(g.dropped, a.ID)
return nil
}
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
g.mu.Lock()
g.n++
g.mu.Unlock()
return g.why(a), nil
}
func (g *goneAsks) asked() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.n
}
// waitForAsks waits until n asks were put.
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
}
}
return r.asks.Asks()
}
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
// again; the key is done when a node takes it.
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
t.Run("waits in its stream: never asked again", func(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
waitForAsks(t, r, 1)
time.Sleep(100 * time.Millisecond)
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
t.Fatalf("the key = %+v while its ask waits, want building", v)
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no ask was put")
}
if gone.asked() == 0 {
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
}
r.asks.End(r.answer(r.asks.Asks()[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
})
for _, c := range []struct {
name string
askings int
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
step string
byHand bool
always bool // Gone says every ask cannot be answered, not only the first
puts int
done bool
asking int // the asking the event says, 0 when none is said
}{
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
} {
t.Run(c.name, func(t *testing.T) { // 1945
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Askings = c.askings
var said []AskedAgain
var mu sync.Mutex
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
first := ""
if c.step != "" {
first = "mvl0frv18d1960"
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
r.st.vals[r.key] = data
}
gone := &goneAsks{}
if !c.byHand {
gone.why = func(a delivery.StepAsk) string {
if c.always || a.ID == first || first == "" {
return "it waits first in line while a node took a later ask"
}
return ""
}
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
if c.byHand {
waitForAsks(t, r, 1)
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no lane of the key waits")
}
}
}
if c.done {
asks := waitForAsks(t, r, c.puts)
if first == "" && !c.byHand {
first = asks[0].ID
}
last := asks[len(asks)-1]
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
}
r.asks.End(r.answer(last))
}
r.b.Wait()
v := valueOf(t, r.st, r.key)
if n := len(r.asks.Asks()); n != c.puts {
t.Errorf("%d asks put, want %d", n, c.puts)
}
if c.done && v.State != delivery.BuildDone {
t.Errorf("the key = %+v, want done from the ask asked again", v)
}
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
}
mu.Lock()
defer mu.Unlock()
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
}
gone.mu.Unlock()
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
t.Errorf("said %+v, want once at asking %d", said, c.asking)
}
})
}
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
taken := func(kind string, at time.Time) *Taken {
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
}
later := taken(delivery.AskDelivery, put.Add(time.Minute))
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
ahead := func(n int, step string) []Queued {
var qs []Queued
for i := range n {
at := put.Add(-time.Duration(n-i) * time.Minute)
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
}
return qs
}
seen := func(qs []Queued, last *Taken) AskSeen {
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
}
for _, c := range []struct {
name string
seen AskSeen
want string // a part of the why, "" when it can be answered
waiting int
}{
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
{"first in line, nothing taken since", seen(nil, nil), "", 0},
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
} {
got := c.seen.Why()
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
}
}
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
t.Errorf("AskedAt does not read an ask id's time")
}
})
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
state := func(vals map[string]string) StateRead {
return StateRead{Keys: func() ([]string, error) {
var ks []string
for k := range vals {
ks = append(ks, k)
}
return ks, nil
}, Get: func(k string) (json.RawMessage, time.Time, error) {
if v, ok := vals[k]; ok {
return json.RawMessage(v), at, nil
}
return nil, time.Time{}, nil
}}
}
key, _ := delivery.BuilderBuildKey("m", fp1)
done, _ := delivery.BuilderBuildKey("d", fp1)
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Now: func() time.Time { return at.Add(time.Hour) }}
out, err := r.Stuck("")
if err != nil {
t.Fatal(err)
}
stuck := out["stuck"].([]Stuck)
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
}
a := stuck[0].Asks
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
}
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
// only for the rows kept.
var gets, written, inFlight, maxFlight atomic.Int64
vals := map[string]string{}
for i := 0; i < 3000; i++ {
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
vals[k] = string(finished)
if (i/100)%10 == 0 {
vals[k] = string(building)
}
}
slow := state(vals)
get := slow.Get
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
gets.Add(1)
n := inFlight.Add(1)
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
}
time.Sleep(time.Millisecond)
inFlight.Add(-1)
raw, _, err := get(k)
return raw, time.Time{}, err
}
slow.Written = func(string) time.Time { written.Add(1); return at }
for _, row := range []struct {
module string
stuck int
gets int64
cut bool
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
gets.Store(0)
written.Store(0)
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
if err != nil {
t.Fatal(err)
}
_, cut := out["cut"]
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
}
}
if maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time")
}
})
time.Sleep(100 * time.Millisecond)
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
t.Fatalf("the key = %+v while its ask waits, want building", v)
}
r.asks.End(r.answer(r.asks.Asks()[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
}
// heldAsks says an ask is held when its id is among held.
@@ -1,290 +0,0 @@
package build
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
// builder's state reasks, which the builder watches and answers there.
const (
StuckTool = "builder_stuck"
AskAgainTool = "builder_ask_again"
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
ReAsksState = "reasks"
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
)
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
// read it, its answer.
type ReAsk struct {
V int `json:"v"`
Why string `json:"why"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
var r ReAsk
if err := json.Unmarshal(data, &r); err != nil {
return r, err
}
if r.V != 1 {
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
}
return r, nil
}
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the build is asked again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
Keys func() ([]string, error)
Get func(key string) (json.RawMessage, time.Time, error)
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
// only for a row they keep, and only when Get did not say.
Written func(key string) time.Time
}
// StuckAsk is one ask a key waits on.
type StuckAsk struct {
Language string `json:"language"`
BuildModule string `json:"build_module"`
ID string `json:"id"`
Asking int `json:"asking"`
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
// state on its node since its start.
Asked time.Time `json:"asked,omitzero"`
Step string `json:"step"`
}
// Stuck is one build key that waits on an ask.
type Stuck struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp"`
State string `json:"state"`
Step string `json:"step"`
By string `json:"by"`
Since time.Time `json:"since,omitzero"`
Age string `json:"age,omitempty"`
Asks []StuckAsk `json:"asks"`
ReAsk *ReAsk `json:"reask,omitempty"`
Unasked bool `json:"unasked,omitempty"`
NoneStep int `json:"asks_with_no_step"`
}
// StuckReader reads what builder_stuck answers.
type StuckReader struct {
Builds StateRead
ReAsks StateRead
// Steps are the build modules' steps states, by build module.
Steps map[string]StateRead
Now func() time.Time
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
Logf func(format string, args ...any)
}
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
const (
ReadWorkers = 16
MaxReads = 1000
)
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
for w := 0; w < min(ReadWorkers, n); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
}
wg.Wait()
return first
}
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
func (r StuckReader) Stuck(module string) (map[string]any, error) {
started := time.Now()
reads := 0
defer func() {
if r.Logf != nil {
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
}
}()
keys, err := r.Builds.Keys()
if err != nil {
return nil, err
}
now := time.Now
if r.Now != nil {
now = r.Now
}
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
stepKeys := map[string]map[string]string{}
for bm, s := range r.Steps {
ks, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
}
byID := make(map[string]string, len(ks))
for _, k := range ks {
if _, id, err := delivery.ParseStepKey(k); err == nil {
byID[id] = k
}
}
stepKeys[bm] = byID
}
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
// parallel, at most MaxReads of them.
type parsed struct{ key, mod, fp string }
var matching []parsed
for _, key := range keys {
mod, fp, err := delivery.ParseBuilderBuildKey(key)
if err != nil || (module != "" && mod != module) {
continue
}
matching = append(matching, parsed{key, mod, fp})
}
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
cut := len(matching) > MaxReads
total := len(matching)
if cut {
matching = matching[:MaxReads]
}
values := make([]json.RawMessage, len(matching))
ats := make([]time.Time, len(matching))
if err := parallel(len(matching), func(i int) error {
raw, at, err := r.Builds.Get(matching[i].key)
if err == nil {
values[i], ats[i] = raw, at
}
return nil
}); err != nil {
return nil, err
}
reads += len(matching)
out := []Stuck{}
for i, m := range matching {
key, mod, fp := m.key, m.mod, m.fp
raw, at := values[i], ats[i]
if raw == nil {
continue
}
v, err := delivery.DecodeBuilderBuild(raw)
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
continue
}
if at.IsZero() && r.Builds.Written != nil {
at = r.Builds.Written(key)
}
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
if !at.IsZero() {
s.Age = now().Sub(at).Round(time.Second).String()
}
for language, id := range v.Asks {
bm, _ := delivery.BuildModuleOf(language)
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
if t, ok := idTime(id); ok {
a.Asked = t
}
if st, ok := r.Steps[bm]; ok {
if k, ok := stepKeys[bm][id]; ok {
reads++
if raw, _, err := st.Get(k); err == nil && raw != nil {
if step, err := delivery.DecodeStep(raw); err == nil {
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
}
}
}
}
if a.Step == "none" {
s.NoneStep++
}
s.Asks = append(s.Asks, a)
}
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
if r.ReAsks.Get != nil {
reads++
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
if ra, err := DecodeReAsk(raw); err == nil {
s.ReAsk = &ra
}
}
}
out = append(out, s)
}
sort.Slice(out, func(i, j int) bool {
if !out[i].Since.Equal(out[j].Since) {
return out[i].Since.Before(out[j].Since)
}
return out[i].Key < out[j].Key
})
answer := map[string]any{"stuck": out, "count": len(out),
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
if cut {
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
}
return answer, nil
}
// StuckBuildModules are the build modules whose steps builder_stuck reads.
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
func idTime(id string) (time.Time, bool) {
if len(id) <= 6 {
return time.Time{}, false
}
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
if err != nil || ms <= 0 {
return time.Time{}, false
}
return time.UnixMilli(ms).UTC(), true
}
+11 -26
View File
@@ -23,21 +23,18 @@ type state struct {
}
type artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
}
type manifest struct {
Module string `json:"module"`
Version string `json:"version"`
Slug string `json:"slug"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
Consumes []string `json:"consumes"`
State []state `json:"state"`
Reads []string `json:"reads"`
@@ -87,7 +84,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -115,14 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.SecretsOwner != "mesh-delivery-builder" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
t.Errorf("emits = %v, want %v", m.Emits, want)
}
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the builder has none", k)
}
@@ -137,7 +127,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
procs = append(procs, r)
}
if r["type"] == "tool" || r["type"] == "seat" {
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
}
}
if len(procs) != 1 {
@@ -159,13 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
t.Errorf("the tools' source %s: %v", a.From, err)
if len(m.Build.Artifacts) != 1 {
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
+1 -22
View File
@@ -2,14 +2,6 @@
"module": "mesh-delivery-builder",
"version": "1",
"slug": "builder",
"tools": [
"builder_ask_again",
"builder_seat_queue",
"builder_stuck"
],
"emits": [
"build.asked-again"
],
"consumes": [
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported",
@@ -20,9 +12,7 @@
],
"state": [
{ "name": "builds", "history": 5 },
{ "name": "instances", "history": 1 },
{ "name": "queues", "history": 1 },
{ "name": "reasks", "history": 1 }
{ "name": "instances", "history": 1 }
],
"reads": [
"mesh-build-go.steps",
@@ -88,17 +78,6 @@
"system": "arch",
"from": "cmd/mesh-delivery-builder",
"binary": "mesh-delivery-builder"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-builder-tools",
"binary": "mesh-delivery-builder-tools",
"loads": [
"mesh-delivery-builder-tools"
]
}
]
}
@@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"os"
"sync/atomic"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -16,35 +15,8 @@ import (
"mesh-delivery-checker/internal/check"
)
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
var reads atomic.Int64
// cutAnswer says in the answer that it was cut, and how to narrow it.
func cutAnswer(answer map[string]any, cut string) map[string]any {
if cut != "" {
answer["cut"], answer["note"] = true, cut
}
return answer
}
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
func logged(tools []stdio.Tool) []stdio.Tool {
for i := range tools {
name, run := tools[i].Name, tools[i].Run
tools[i].Run = func(args map[string]any) (any, error) {
reads.Store(0)
start := time.Now()
out, err := run(args)
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
return out, err
}
}
return tools
}
func read(s stdio.KeptState) check.StateRead {
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
reads.Add(1)
e, err := s.Get(key)
if err != nil || e == nil {
return nil, err
@@ -128,8 +100,8 @@ func Tools(r check.Reader) []stdio.Tool {
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
Run: func(args map[string]any) (any, error) {
group, _ := args["group"].(string)
vs, cut, err := r.Verdicts(group)
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
vs, err := r.Verdicts(group)
return map[string]any{"verdicts": vs}, err
}},
{Name: check.PullReadTool,
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
@@ -150,8 +122,8 @@ func Tools(r check.Reader) []stdio.Tool {
Run: func(args map[string]any) (any, error) {
level, _ := args["level"].(string)
digest, _ := args["digest"].(string)
kept, cut, err := r.Kept(level, digest)
return cutAnswer(map[string]any{"kept": kept}, cut), err
kept, err := r.Kept(level, digest)
return map[string]any{"kept": kept}, err
}},
{Name: check.PreviewsReadTool,
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
@@ -166,8 +138,8 @@ func Tools(r check.Reader) []stdio.Tool {
if head != "" && len(head) < 7 {
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
}
ps, bad, cut, err := r.Previews(repository, head)
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
ps, bad, err := r.Previews(repository, head)
return map[string]any{"previews": ps, "unreadable": bad}, err
}},
{Name: check.StatusReadTool,
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
@@ -212,7 +184,7 @@ func main() {
}},
Trunks: trunks,
}
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
}
@@ -152,12 +152,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
buckets := map[string]jetstream.KeyValue{}
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil {
if buckets[name], err = bucket(name); err != nil {
return err
}
}
@@ -207,7 +212,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again.
var builds check.BuildStates
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
default:
builds = buildReader{kvStore{kv}}
}
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
@@ -280,38 +290,47 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
// issue 1912), so the node-engine's have is not watched here.
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil {
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil {
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again")
} else {
go func() {
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
}
})
}()
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
default:
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
}
})
}()
}
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
// yet no trunk is known, so nothing is checked.
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil {
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey {
trunks.Put(e.Value(), !opOf(e))
}
})
if err != nil {
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
} else {
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
}
})
}()
return err
}
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
}
})
}()
}
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
@@ -193,9 +193,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
@@ -3,19 +3,11 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -97,51 +89,6 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
@@ -8,8 +8,6 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -65,14 +63,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
return nil
return err
}
w.watched[module] = true
go func() {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,79 +0,0 @@
package check
import (
"fmt"
"sort"
"sync"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
// through mesh-sdk (rules-are-weighed).
const ReadWorkers = 16
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
const MaxReads = 1000
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
// further read starts.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
for w := 0; w < min(ReadWorkers, n); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
}
wg.Wait()
return first
}
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
func capKeys(keys []string, narrow string) ([]string, string) {
if len(keys) <= MaxReads {
return keys, ""
}
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
}
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
func newestFirst(keys []string) {
id := func(k string) (string, bool) {
_, id, err := delivery.ParseStepKey(k)
return id, err == nil
}
sort.Slice(keys, func(i, j int) bool {
a, aok := id(keys[i])
b, bok := id(keys[j])
switch {
case aok != bok:
return aok
case len(a) != len(b):
return len(a) > len(b)
case a != b:
return a > b
}
return keys[i] > keys[j]
})
}
@@ -89,46 +89,37 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
return out, nil
}
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
return nil, fmt.Errorf("the verdicts state cannot be read here")
}
keys, err := r.VerdictState.Keys()
if err != nil {
return nil, "", err
return nil, err
}
sort.Strings(keys)
var matching []string
var out []VerdictRead
for _, k := range keys {
if group != "" {
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
continue
}
}
matching = append(matching, k)
}
matching, cut = capKeys(matching, "give a group to narrow the answer")
rows := make([]VerdictRead, len(matching))
found := make([]bool, len(matching))
if err := parallel(len(matching), func(i int) error {
raw, err := r.VerdictState.Get(matching[i])
if err != nil || len(raw) == 0 {
return err
raw, err := r.VerdictState.Get(k)
if err != nil {
return nil, err
}
rows[i], err = r.read(matching[i], raw)
found[i] = err == nil
return err
}); err != nil {
return nil, "", err
}
for i, row := range rows {
if found[i] {
out = append(out, row)
if len(raw) == 0 {
continue
}
vr, err := r.read(k, raw)
if err != nil {
return nil, err
}
out = append(out, vr)
}
return out, cut, nil
return out, nil
}
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
@@ -142,95 +133,74 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
return VerdictRead{}, err
}
sort.Strings(keys)
keys, cut := capKeys(keys, "")
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
for from := 0; from < len(keys); from += ReadWorkers * 4 {
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
cands := make([]delivery.Candidate, len(batch))
ok := make([]bool, len(batch))
_ = parallel(len(batch), func(i int) error {
raw, err := r.CandidateState.Get(batch[i])
if err != nil || len(raw) == 0 {
return nil
}
c, err := delivery.DecodeCandidate(raw)
cands[i], ok[i] = c, err == nil
return nil
})
for i, c := range cands {
if !ok[i] {
for _, k := range keys {
raw, err := r.CandidateState.Get(k)
if err != nil || len(raw) == 0 {
continue
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
continue
}
for _, p := range c.Pulls {
if p.Repository != repository || p.Number != number {
continue
}
for _, p := range c.Pulls {
if p.Repository != repository || p.Number != number {
continue
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return VerdictRead{}, err
}
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return VerdictRead{}, err
}
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
}
}
if cut != "" {
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
}
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
}
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
return nil, "", fmt.Errorf("the kept state cannot be read here")
return nil, fmt.Errorf("the kept state cannot be read here")
}
if digest != "" {
key, err := delivery.KeptKey(level, digest)
if err != nil {
return nil, "", err
return nil, err
}
raw, err := r.KeptState.Get(key)
if err != nil {
return nil, "", err
return nil, err
}
if len(raw) == 0 {
return nil, "", fmt.Errorf("no kept verdict %s", key)
return nil, fmt.Errorf("no kept verdict %s", key)
}
return map[string]json.RawMessage{key: raw}, "", nil
return map[string]json.RawMessage{key: raw}, nil
}
keys, err := r.KeptState.Keys()
if err != nil {
return nil, "", err
return nil, err
}
sort.Strings(keys)
var matching []string
for _, k := range keys {
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
matching = append(matching, k)
}
}
matching, cut := capKeys(matching, "give a level and a digest to read one")
values := make([]json.RawMessage, len(matching))
if err := parallel(len(matching), func(i int) (err error) {
values[i], err = r.KeptState.Get(matching[i])
return err
}); err != nil {
return nil, "", err
}
out := map[string]json.RawMessage{}
for i, k := range matching {
if len(values[i]) > 0 {
out[k] = values[i]
for _, k := range keys {
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
continue
}
raw, err := r.KeptState.Get(k)
if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
}
}
return out, cut, nil
return out, nil
}
// PreviewRead is one preview as stored, with the parts of its key.
@@ -244,18 +214,17 @@ type PreviewRead struct {
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
return nil, nil, fmt.Errorf("the previews state cannot be read here")
}
keys, err := r.PreviewState.Keys()
if err != nil {
return nil, nil, "", err
return nil, nil, err
}
sort.Strings(keys)
type part struct{ repo, head, trunk string }
parts := map[string]part{}
var matching, bad []string
out := []PreviewRead{}
var bad []string
for _, k := range keys {
repo, h, trunk, err := delivery.ParsePreviewKey(k)
if err != nil {
@@ -265,25 +234,15 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, stri
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
continue
}
parts[k] = part{repo, h, trunk}
matching = append(matching, k)
}
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
values := make([]json.RawMessage, len(matching))
if err := parallel(len(matching), func(i int) (err error) {
values[i], err = r.PreviewState.Get(matching[i])
return err
}); err != nil {
return nil, nil, "", err
}
out := []PreviewRead{}
for i, k := range matching {
if len(values[i]) > 0 {
p := parts[k]
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
raw, err := r.PreviewState.Get(k)
if err != nil {
return nil, nil, err
}
if len(raw) > 0 {
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
}
}
return out, bad, cut, nil
return out, bad, nil
}
// RunningStep is a step in flight on a build module.
@@ -341,28 +300,16 @@ func (r Reader) Status() (Status, error) {
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
continue
}
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
newestFirst(keys)
keys, cut := capKeys(keys, "")
if cut != "" {
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
}
steps := make([]delivery.Step, len(keys))
isRunning := make([]bool, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := sr.Get(keys[i])
sort.Strings(keys)
for _, k := range keys {
raw, err := sr.Get(k)
if err != nil || len(raw) == 0 {
return nil
}
step, err := delivery.DecodeStep(raw)
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
return nil
})
for i := range keys {
if !isRunning[i] {
continue
}
step := steps[i]
step, err := delivery.DecodeStep(raw)
if err != nil || step.State != delivery.StepRunning {
continue
}
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
if !step.Started.IsZero() {
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
@@ -397,41 +344,23 @@ func (r Reader) Status() (Status, error) {
return st, err
}
sort.Strings(keys)
total := len(keys)
keys, cut := capKeys(keys, "")
if cut != "" {
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
}
type judged struct {
c delivery.Candidate
raw json.RawMessage
ok bool
}
rows := make([]judged, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := r.CandidateState.Get(keys[i])
for _, k := range keys {
raw, err := r.CandidateState.Get(k)
if err != nil || len(raw) == 0 {
return nil
continue
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
return nil
continue
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return nil
}
rows[i].c = c
if vraw, err := r.VerdictState.Get(vkey); err == nil {
rows[i].raw, rows[i].ok = vraw, true
}
return nil
})
for _, row := range rows {
if !row.ok {
continue
}
c, raw := row.c, row.raw
raw, err = r.VerdictState.Get(vkey)
if err != nil {
continue
}
if len(raw) == 0 {
st.Unjudged = append(st.Unjudged, c.Group)
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
@@ -3,9 +3,7 @@ package check
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
@@ -53,7 +51,7 @@ func readerOf(t *testing.T) (Reader, string) {
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
rd, key := readerOf(t)
vs, _, err := rd.Verdicts("")
vs, err := rd.Verdicts("")
if err != nil || len(vs) != 1 || vs[0].Key != key {
t.Fatalf("verdicts = %+v, %v", vs, err)
}
@@ -63,7 +61,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
t.Errorf("reports = %v", vs[0].Reports)
}
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
t.Errorf("another group's verdicts = %+v", only)
}
}
@@ -87,11 +85,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
rd, _ := readerOf(t)
all, _, err := rd.Kept("", "")
all, err := rd.Kept("", "")
if err != nil || len(all) != 1 {
t.Fatalf("kept = %v, %v", all, err)
}
one, _, err := rd.Kept("unit", digA)
one, err := rd.Kept("unit", digA)
if err != nil || len(one) != 1 {
t.Fatalf("one kept = %v, %v", one, err)
}
@@ -100,10 +98,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
t.Errorf("kept value = %+v, %v", k, err)
}
}
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
t.Errorf("integration = %v", none)
}
if _, _, err := rd.Kept("unit", digB); err == nil {
if _, err := rd.Kept("unit", digB); err == nil {
t.Error("a digest nothing kept was answered")
}
}
@@ -117,17 +115,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
}
key, _ := delivery.PreviewKey(repoA, head1, head2)
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
got, bad, _, err := rd.Previews(repoA, head1[:8])
got, bad, err := rd.Previews(repoA, head1[:8])
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
t.Fatalf("previews = %+v, %v", got, err)
}
if len(bad) != 1 || bad[0] != "junk" {
t.Errorf("unreadable = %v", bad)
}
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
t.Errorf("another repository's = %+v", got)
}
if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
if _, _, err := (Reader{}).Previews("", ""); err == nil {
t.Error("answered with no state to read")
}
}
@@ -235,121 +233,3 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
}
}
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
type slowBucket struct {
vals fakeState
reads, lists atomic.Int64
inFlight, maxFlight atomic.Int64
}
func (b *slowBucket) get(key string) (json.RawMessage, error) {
b.reads.Add(1)
now := b.inFlight.Add(1)
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
}
time.Sleep(time.Millisecond)
b.inFlight.Add(-1)
return b.vals.read().Get(key)
}
func (b *slowBucket) read() StateRead {
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
}
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
func (b *slowBucket) Create(string, []byte) error { return nil }
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
kept := fakeState{}
for i := 0; i < 1800; i++ {
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
kept[k] = []byte(`{"v":1}`)
}
previews := fakeState{}
for i := 0; i < 1500; i++ {
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
previews[k] = []byte(`{"v":1}`)
}
for _, row := range []struct {
name string
call func(*slowBucket) (rows int, cut string, err error)
rows int
reads, lists int64
cut bool
}{
{"kept of one key", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
return len(m), c, err
}, 1, 1, 0, false},
{"kept of a level", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
return len(m), c, err
}, 200, 200, 1, false},
{"kept of everything", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
return len(m), c, err
}, MaxReads, MaxReads, 1, true},
{"previews of a repository", func(b *slowBucket) (int, string, error) {
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
return len(p), c, err
}, 1200, 1000, 1, true},
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
return len(p), c, err
}, 1, 1, 1, false},
} {
b := &slowBucket{}
b.vals = kept
if strings.HasPrefix(row.name, "previews") {
b.vals = previews
}
start := time.Now()
rows, cut, err := row.call(b)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
}
if row.reads > 20 && b.maxFlight.Load() < 2 {
t.Errorf("%s: values were read one at a time", row.name)
}
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
}
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
steps := fakeState{}
for i := 0; i < 1500; i++ {
id := fmt.Sprintf("mvo5%04dabcdef", i)
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
if i == 1499 {
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
}
raw, _ := delivery.EncodeStep(step)
steps["g14."+id] = raw
}
sb := &slowBucket{vals: steps}
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
Now: func() time.Time { return now }}.Status()
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
}
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
tries := fakeState{}
for i := 0; i < 1500; i++ {
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
}
tb := &slowBucket{vals: tries}
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
}
}
@@ -29,8 +29,6 @@ const (
TriesBucket = delivery.CheckerModule + "_" + TriesState
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
MaxTries = 4
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
TryScan = 64
// TryKeep is how long an ended try's key is kept before the checker removes it.
TryKeep = 7 * 24 * time.Hour
)
@@ -157,33 +155,15 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if err != nil {
return "", fmt.Errorf("the tries cannot be listed: %w", err)
}
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last).
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) != len(keys[j]) {
return len(keys[i]) > len(keys[j])
}
return keys[i] > keys[j]
})
if len(keys) > TryScan {
keys = keys[:TryScan]
}
tries := make([]Try, len(keys))
live := make([]bool, len(keys))
if err := parallel(len(keys), func(i int) error {
raw, err := store.Get(keys[i])
sort.Strings(keys)
var open []string
for _, k := range keys {
raw, err := store.Get(k)
if err != nil || len(raw) == 0 {
return nil
continue
}
cur, err := DecodeTry(raw)
tries[i], live[i] = cur, err == nil && !cur.Over()
return nil
}); err != nil {
return "", err
}
var open []string
for i, cur := range tries {
if !live[i] {
if err != nil || cur.Over() {
continue
}
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
@@ -22,8 +22,6 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -190,18 +188,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
call := callerOn(conn, cred.User)
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
var turn sync.Mutex
kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream")
kv, err := js.KeyValue(ctx, headsBucket)
if err != nil {
return err
return fmt.Errorf("the bucket %s: %w", headsBucket, err)
}
heads := kvHeads{kv}
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
rec := &recorder{pub: js, heads: heads, logger: logger}
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once")
performedKV, err := js.KeyValue(ctx, performedBucket)
if err != nil {
return err
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
}
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
watching := make(chan error, 3)
@@ -209,12 +207,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
bucket string
react func(ctx context.Context, key string, rev uint64, value []byte)
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
if kv == nil {
kv, err := js.KeyValue(ctx, w.bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
// The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
// by its delivery when the gate arrives.
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
continue
}
if err != nil {
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
}
go func() {
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
@@ -229,8 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go onReconnect(ctx, again, said, logger)
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
if err != nil {
// Essential: the forge's events are what the bridge carries.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
}
cc, err := cons.Consume(func(m jetstream.Msg) {
hear(&turn, func() {
@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"slices"
"strings"
@@ -13,8 +12,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -69,16 +66,6 @@ func body(m map[string]any) []byte {
}
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
@@ -243,8 +230,3 @@ func someFiles(_, _, path, _ string) ([]byte, error) {
}
return nil, bridge.ErrNotFound
}
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -240,21 +240,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil {
return err
}
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint")
modules, err := bucket(delivery.ModulesBucket)
if err != nil {
return err
}
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest")
manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
builds, err := bucket(delivery.BuildsBucket)
if err != nil {
return err
}
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
// such a module waits for its path, every other is indexed (issue 1945).
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
known := index.NewBuilds()
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
@@ -265,14 +273,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
known.Delete(e.Key())
}
}
errs := make(chan error, 1)
if builds != nil {
if w, err := settle(ctx, builds, each); err != nil {
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
} else {
go func() { errs <- follow(ctx, w, each) }()
}
w, err := settle(ctx, builds, each)
if err != nil {
return err
}
errs := make(chan error, 1)
go func() { errs <- follow(ctx, w, each) }()
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
account, err := process.ReadAccount(path)
@@ -290,17 +296,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
sourceErrs := make(chan error, 1)
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
}
}
if sw, err := settle(ctx, source, each); err != nil {
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again")
} else {
go func() { sourceErrs <- follow(ctx, sw, each) }()
sw, err := settle(ctx, source, each)
if err != nil {
return err
}
go func() { sourceErrs <- follow(ctx, sw, each) }()
}
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
@@ -370,8 +381,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
if err != nil {
// Essential: the forge's merges are what the indexer indexes.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
}
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
@@ -3,7 +3,7 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"fmt"
"log"
"strings"
"sync"
@@ -55,14 +55,11 @@ func (w *previewWatch) ensure() error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
return nil
return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
return nil
return err
}
w.watched = true
go func() {
@@ -193,9 +193,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
@@ -10,8 +10,6 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -75,14 +73,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
return nil
return err
}
w.watched[module] = true
go func() {
@@ -3,7 +3,6 @@ package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"sync"
@@ -22,7 +21,6 @@ type laterBuckets struct {
mu sync.Mutex
appears map[string]jetstream.KeyValue // buckets that exist now
asked map[string]int
refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945)
}
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
@@ -32,9 +30,6 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa
if kv, ok := l.appears[name]; ok {
return kv, nil
}
if l.refuse != nil {
return nil, l.refuse
}
return nil, jetstream.ErrBucketNotFound
}
@@ -105,12 +100,6 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
}
var logs bytes.Buffer
// 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running.
refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")}
if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") {
t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String())
}
logs.Reset()
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
t.Fatal(err)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -22,8 +22,6 @@ import (
"strings"
"syscall"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -121,14 +119,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
}
if err != nil {
// Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant).
return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err)
return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err)
}
// The answers it writes and what it asked: its own work, so each stops the start, naming its grant.
var kvs [2]jetstream.KeyValue
for i, name := range []string{delivery.AnswersBucket, askedBucket} {
if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil {
return err
if kvs[i], err = js.KeyValue(ctx, name); err != nil {
return fmt.Errorf("the bucket %s: %w", name, err)
}
}
state := kvStore{ctx: ctx, kv: kvs[1]}
@@ -144,7 +140,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
failed := make(chan error, 2)
assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf)
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
if viewKV != nil {
t.View = kvGate{ctx: ctx, kv: viewKV}
@@ -176,8 +172,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
// teller is what an optional bucket of the teller leaves out.
const teller = "the asks and answers it holds are not carried until the teller is started again with it"
// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not.
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
kv, err := js.KeyValue(ctx, name)
if err != nil {
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
return nil
}
return kv
}
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
@@ -1,17 +1,12 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
@@ -241,16 +236,6 @@ func (b *bed) askedOf(subject string) []published {
}
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
@@ -533,8 +518,3 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
}
}
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -6,8 +6,6 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -9,9 +9,8 @@ import (
"os"
"strings"
"mesh-delivery-view/internal/view"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-view/internal/view"
)
// Tools are the read tools over a modules state.
@@ -86,8 +85,7 @@ func Tools(r view.Reader) []stdio.Tool {
}},
{Name: view.WantsReadTool,
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given); a node and a module read that one key. " +
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
@@ -102,8 +100,7 @@ func Tools(r view.Reader) []stdio.Tool {
}},
{Name: view.DifferencesReadTool,
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module (both: that one key). " +
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
@@ -111,8 +108,7 @@ func Tools(r view.Reader) []stdio.Tool {
}},
{Name: view.HavesReadTool,
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
"and <node>.node (the node's own facts). Narrow by node and module (both: that one key). " +
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
"and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
@@ -4,9 +4,7 @@ import (
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"mesh-delivery-view/internal/view"
)
@@ -166,7 +164,7 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
type bigBuilds struct {
fake fake
reads *atomic.Int64
reads *int
}
func newBig(n int) bigBuilds {
@@ -174,15 +172,15 @@ func newBig(n int) bigBuilds {
for i := 0; i < n; i++ {
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
}
return bigBuilds{fake: f, reads: new(atomic.Int64)}
return bigBuilds{fake: f, reads: new(int)}
}
func (b bigBuilds) reader() view.Reader {
r := b.fake.reader()
get := r.State.Get
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { b.reads.Add(1); return get(k) },
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
GetEntry: func(k string) (json.RawMessage, uint64, error) {
b.reads.Add(1)
*b.reads++
raw, err := get(k)
var rev uint64
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
@@ -200,8 +198,8 @@ func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
t.Fatal(err)
}
a := out.(view.BuildsAnswer)
if b.reads.Load() != 20 {
t.Errorf("read %d values for a module of 20 keys, want 20", b.reads.Load())
if *b.reads != 20 {
t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
}
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
@@ -224,13 +222,13 @@ func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
t.Fatal(err)
}
a := out.(view.BuildsAnswer)
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || b.reads.Load() > view.DefaultBuildsLimit {
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, b.reads.Load())
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
}
b = newBig(2000)
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
if a := out.(view.BuildsAnswer); b.reads.Load() != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", b.reads.Load(), view.MaxBuildReads, a.Cut)
if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
}
}
@@ -282,41 +280,6 @@ func keptReader() view.Reader {
}
}
// slowBucket is a bucket of the live shape (2026-10-25: node_haves and assigner_wants timed out at 30 s): keys over 4 nodes
// and perNode modules each, written by key(node, module), whose Get takes delay and counts the values read, the lists of
// the bucket and the most Gets in flight at once.
type slowBucket struct {
fake fake
delay time.Duration
reads, lists atomic.Int64
inFlight, maxFlight atomic.Int64
}
func newSlow(perNode int, delay time.Duration, key func(node string, i int) string) *slowBucket {
b := &slowBucket{fake: fake{}, delay: delay}
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
for i := 0; i < perNode; i++ {
b.fake[key(n, i)] = `{"fp":"x"}`
}
}
return b
}
func (b *slowBucket) state() view.StateRead {
f := b.fake.reader().State
return view.StateRead{
Keys: func() ([]string, error) { b.lists.Add(1); return f.Keys() },
Get: func(k string) (json.RawMessage, error) {
b.reads.Add(1)
now := b.inFlight.Add(1)
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
}
time.Sleep(b.delay)
b.inFlight.Add(-1)
return f.Get(k)
}}
}
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
r := keptReader()
out, err := run(t, r, view.WantsReadTool, map[string]any{})
@@ -335,39 +298,6 @@ func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
t.Errorf("module zsh = %v", got)
}
// On the live shape (4 nodes, 150 wants each, 1 ms a Get): a node and a module read that one key and list nothing;
// a node reads only its own keys; the whole answer reads its values in parallel.
slow := newSlow(150, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("want.%s.m%03d", n, i) })
r = view.Reader{WantsState: slow.state()}
for _, row := range []struct {
args map[string]any
wants int
reads, lists int64
}{
{map[string]any{"node": "ace", "module": "m007"}, 1, 1, 0},
{map[string]any{"node": "ace", "module": "gone"}, 0, 1, 0},
{map[string]any{"node": "g14"}, 150, 150, 1},
{map[string]any{"module": "m007"}, 4, 4, 1},
{map[string]any{}, 600, 600, 1},
} {
slow.reads.Store(0)
slow.lists.Store(0)
start := time.Now()
out, err := run(t, r, view.WantsReadTool, row.args)
took := time.Since(start)
if err != nil {
t.Fatalf("%v: %v", row.args, err)
}
a := out.(view.WantsAnswer)
if len(a.Wants) != row.wants || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut {
t.Errorf("%v: %d wants, %d reads, %d lists, cut %v; want %d, %d, %d", row.args, len(a.Wants), slow.reads.Load(), slow.lists.Load(), a.Cut, row.wants, row.reads, row.lists)
}
t.Logf("assigner_wants %v: %d values in %v (one after the other: %v)", row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
}
if slow.maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
}
}
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
@@ -398,39 +328,4 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
t.Errorf("haves of ace (the node's own key too) = %v", got)
}
// More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list
// nothing; the whole answer reads MaxStateReads values in parallel and says it was cut.
slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) })
r = view.Reader{HavesState: slow.state(), DifferencesState: slow.state()}
for _, row := range []struct {
tool string
args map[string]any
keys int
reads, lists int64
cut bool
}{
{view.HavesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
{view.DifferencesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
{view.HavesReadTool, map[string]any{"node": "ace"}, 300, 300, 1, false},
{view.HavesReadTool, map[string]any{"module": "m042"}, 4, 4, 1, false},
{view.HavesReadTool, map[string]any{}, view.MaxStateReads, view.MaxStateReads, 1, true},
} {
slow.reads.Store(0)
slow.lists.Store(0)
start := time.Now()
out, err := run(t, r, row.tool, row.args)
took := time.Since(start)
if err != nil {
t.Fatalf("%s %v: %v", row.tool, row.args, err)
}
a := out.(view.KeptAnswer)
if len(a.Keys) != row.keys || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut != row.cut || (a.Cut && !strings.Contains(a.Note, "narrow")) {
t.Errorf("%s %v: %d keys, %d reads, %d lists, cut %v %q; want %d, %d, %d, %v", row.tool, row.args, len(a.Keys), slow.reads.Load(), slow.lists.Load(), a.Cut, a.Note, row.keys, row.reads, row.lists, row.cut)
}
t.Logf("%s %v: %d values in %v (one after the other: %v)", row.tool, row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
}
if slow.maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81 h1:LhljEtt7FbNpVn67eOQbdsFDS8Qj6WPUF6l9IXr9mo4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+53 -203
View File
@@ -10,8 +10,6 @@ import (
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -72,13 +70,12 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
return nil, err
}
sort.Strings(keys)
values, err := r.State.values(keys)
if err != nil {
return nil, err
}
out := map[string]json.RawMessage{}
for i, k := range keys {
raw := values[i]
for _, k := range keys {
raw, err := r.State.Get(k)
if err != nil {
return nil, err
}
if len(raw) == 0 {
continue
}
@@ -132,14 +129,13 @@ func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string,
return nil, nil, err
}
sort.Strings(keys)
values, err := r.CandidatesState.values(keys)
if err != nil {
return nil, nil, err
}
out := map[string]json.RawMessage{}
var unreadable []string
for i, k := range keys {
raw := values[i]
for _, k := range keys {
raw, err := r.CandidatesState.Get(k)
if err != nil {
return nil, nil, err
}
if len(raw) == 0 {
continue
}
@@ -246,51 +242,35 @@ func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, er
}
candidates = append(candidates, k)
}
// The values are read in batches, each batch in parallel and taken in key order: without a module or a state a batch
// is only the rows still missing, so no more is read than the answer needs; otherwise up to readWorkers*4.
reads := 0
walk:
for reads < len(candidates) {
for _, k := range candidates {
if reads >= MaxBuildReads {
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
break
}
size := readWorkers * 4
if prefix == "" && state == "" {
size = limit - len(out.Builds)
}
size = min(size, MaxBuildReads-reads, len(candidates)-reads)
batch := candidates[reads : reads+size]
rows := make([]BuilderRow, len(batch))
found := make([]bool, len(batch))
if err := parallel(len(batch), func(i int) error {
var err error
rows[i], found[i], err = r.builderRow(batch[i])
return err
}); err != nil {
reads++
row, ok, err := r.builderRow(k)
if err != nil {
return BuildsAnswer{}, err
}
reads += len(batch)
for i, row := range rows {
if !found[i] {
if !ok {
continue
}
if state != "" {
var b struct {
State string `json:"state"`
}
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
continue
}
if state != "" {
var b struct {
State string `json:"state"`
}
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
continue
}
}
out.Builds = append(out.Builds, row)
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
if prefix == "" && len(out.Builds) >= limit {
if reads-len(batch)+i+1 < len(candidates) {
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
}
break walk
}
out.Builds = append(out.Builds, row)
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
if prefix == "" && len(out.Builds) >= limit {
if reads < len(candidates) {
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
}
break
}
}
sort.SliceStable(out.Builds, func(i, j int) bool {
@@ -372,9 +352,6 @@ type WantsAnswer struct {
Wants map[string]json.RawMessage `json:"wants"`
Held map[string]json.RawMessage `json:"held"`
Unreadable []string `json:"unreadable,omitempty"`
// Cut says the answer stopped at MaxStateReads values, and Note how to narrow it.
Cut bool `json:"cut,omitempty"`
Note string `json:"note,omitempty"`
}
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
@@ -382,158 +359,60 @@ type WantsAnswer struct {
type KeptAnswer struct {
Keys map[string]json.RawMessage `json:"keys"`
Unreadable []string `json:"unreadable,omitempty"`
Cut bool `json:"cut,omitempty"`
Note string `json:"note,omitempty"`
}
// readWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
// runtime to the bus (the runtime opens the bucket and gets the key: two bus requests), so reading a bucket of a few
// hundred keys one after the other outlasted the 30 s a tool call is given on 2026-10-25 (node_haves, assigner_wants).
const readWorkers = 16
// MaxStateReads bounds the values one call of assigner_wants, assigner_differences or node_haves reads; past it the
// answer says it was cut and how to narrow it.
const MaxStateReads = 1000
// parallel runs read(i) for i in [0, n), at most readWorkers at a time, and answers the first error; once one failed no
// further read starts.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
workers := min(readWorkers, n)
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
if s.Keys == nil || s.Get == nil {
return nil, fmt.Errorf("%s cannot be read here", what)
}
wg.Wait()
return first
}
// values reads the stored value of each key, in parallel, in the order of keys (nil for a key with none).
func (s StateRead) values(keys []string) ([]json.RawMessage, error) {
out := make([]json.RawMessage, len(keys))
err := parallel(len(keys), func(i int) error {
raw, err := s.Get(keys[i])
out[i] = raw
return err
})
return out, err
}
// read reads the values of keys (sorted first) into a map, at most MaxStateReads of them; cut says it stopped there.
func (s StateRead) read(keys []string) (map[string]json.RawMessage, bool, error) {
sort.Strings(keys)
cut := len(keys) > MaxStateReads
if cut {
keys = keys[:MaxStateReads]
}
values, err := s.values(keys)
keys, err := s.Keys()
if err != nil {
return nil, false, err
return nil, err
}
sort.Strings(keys)
out := map[string]json.RawMessage{}
for i, k := range keys {
if len(values[i]) > 0 {
out[k] = values[i]
for _, k := range keys {
raw, err := s.Get(k)
if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
}
}
return out, cut, nil
}
func cutNote(cut bool, n int) string {
if !cut {
return ""
}
return fmt.Sprintf("read the first %d of %d matching keys in key order and stopped; give a node and a module to narrow the answer", MaxStateReads, n)
return out, nil
}
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
// matches).
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
s := r.WantsState
if s.Get == nil {
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
}
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
if node != "" && module != "" {
// One key, read directly: no list of the bucket (a hold has no node, so none is shown).
key, err := delivery.WantKey(node, module)
if module == "node" {
key, err = delivery.WantNodeKey(node)
}
if err != nil {
return WantsAnswer{}, err
}
raw, err := s.Get(key)
if err != nil {
return WantsAnswer{}, err
}
if len(raw) > 0 {
out.Wants[key] = raw
}
return out, nil
}
if s.Keys == nil {
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
}
keys, err := s.Keys()
all, err := r.WantsState.all("the assigner's wants")
if err != nil {
return WantsAnswer{}, err
}
// Narrowed by the key before any value is read.
var matching []string
for _, k := range keys {
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
for k, raw := range all {
if n, m, err := delivery.ParseWantKey(k); err == nil {
if (node == "" || n == node) && (module == "" || m == module) {
matching = append(matching, k)
out.Wants[k] = raw
}
continue
}
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
if n, err := delivery.ParseWantNodeKey(k); err == nil {
if (node == "" || n == node) && (module == "" || module == "node") {
matching = append(matching, k)
out.Wants[k] = raw
}
continue
}
if m, _, err := delivery.ParseHeldKey(k); err == nil {
if node == "" && (module == "" || m == module) {
matching = append(matching, k)
out.Held[k] = raw
}
continue
}
out.Unreadable = append(out.Unreadable, k)
}
sort.Strings(out.Unreadable)
values, cut, err := s.read(matching)
if err != nil {
return WantsAnswer{}, err
}
out.Cut, out.Note = cut, cutNote(cut, len(matching))
for k, raw := range values {
if _, _, err := delivery.ParseHeldKey(k); err == nil {
out.Held[k] = raw
} else {
out.Wants[k] = raw
}
}
return out, nil
}
@@ -557,50 +436,21 @@ func (r Reader) Want(node, module string) (json.RawMessage, error) {
}
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
if s.Get == nil {
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
}
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
if node != "" && module != "" {
// One key, read directly: no list of the bucket.
key, err := delivery.HaveKey(node, module)
if err != nil {
return KeptAnswer{}, err
}
raw, err := s.Get(key)
if err != nil {
return KeptAnswer{}, err
}
if len(raw) > 0 {
out.Keys[key] = raw
}
return out, nil
}
if s.Keys == nil {
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
}
keys, err := s.Keys()
all, err := s.all(what)
if err != nil {
return KeptAnswer{}, err
}
// Narrowed by the key before any value is read.
var matching []string
for _, k := range keys {
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
for k, raw := range all {
n, m, isNode, err := delivery.ParseHaveKey(k)
if err != nil {
out.Unreadable = append(out.Unreadable, k)
continue
}
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
matching = append(matching, k)
out.Keys[k] = raw
}
}
sort.Strings(out.Unreadable)
values, cut, err := s.read(matching)
if err != nil {
return KeptAnswer{}, err
}
out.Keys, out.Cut, out.Note = values, cut, cutNote(cut, len(matching))
return out, nil
}
@@ -22,8 +22,6 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -224,23 +222,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants
// and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build
// seats' steps, and says that moves are not watched (issue 1945).
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue }
for _, b := range []struct {
name string
into *jetstream.KeyValue
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} {
if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil {
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck},
{delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} {
if *b.into, err = bucket(b.name); err != nil {
return err
}
}
kvs.wants = process.OptionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf)
kvs.haves = process.OptionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf)
if kvs.wants == nil || kvs.haves == nil {
kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it
}
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck}, time.Now)
w.Logf(logger.Printf)
@@ -285,15 +283,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent
// until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse.
var haveSeen, wantSeen jetstream.KeyWatcher
if kvs.haves != nil {
if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil {
process.NotWatched(logger.Printf, delivery.HaveBucket, err, "moves are not watched until the watchman starts again")
} else if wantSeen, err = settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }); err != nil {
process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again")
haveSeen.Stop()
haveSeen = nil
}
haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
if err != nil {
return err
}
wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
if err != nil {
return err
}
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
// keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same.
@@ -308,8 +304,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
lang := lang
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) })
if err != nil {
logger.Printf("the steps of %s cannot be watched (%v): its asks are mirrored and never seen taken", module, err)
continue
return err
}
taken[lang] = sw
}
@@ -353,14 +348,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go func() {
errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) })
}()
if haveSeen != nil && wantSeen != nil {
go func() {
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
}()
go func() {
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
}()
}
go func() {
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
}()
go func() {
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
}()
for lang, sw := range taken {
lang, sw := lang, sw
go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }()
@@ -2,17 +2,11 @@ package main
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
)
func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6
@@ -33,23 +27,8 @@ func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6
}
func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "watchman", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
err := run(context.Background(), func(string) string { return "" }, io.Discard)
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err)
}
}
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
-220
View File
@@ -1,220 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
)
// requestWithin bounds one request to the bus: a read the bus refuses is answered by no reply, so the bound is how a
// refusal is told from a slow answer. It is no clock of the part's work: nothing is ever started by it.
const requestWithin = 10 * time.Second
// bucketOf is the bucket a read names: `<module>.<state>` is `<module>_<state>`.
func bucketOf(state string) string {
at := strings.LastIndex(state, ".")
if at <= 0 {
return state
}
return state[:at] + "_" + state[at+1:]
}
// busReader reads other parts' state on the bus, and keeps every value of a state it watches, so a composition reads
// those from memory. A bucket it cannot open (refused, or not there yet) is an error of the call, never the end.
type busReader struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
kvs map[string]jetstream.KeyValue
world map[string]map[string][]byte
settled map[string]bool
}
func newBusReader(ctx context.Context, js jetstream.JetStream) *busReader {
return &busReader{ctx: ctx, js: js, kvs: map[string]jetstream.KeyValue{}, world: map[string]map[string][]byte{},
settled: map[string]bool{}}
}
func (b *busReader) kv(state string) (jetstream.KeyValue, error) {
b.mu.Lock()
kv, ok := b.kvs[state]
b.mu.Unlock()
if ok {
return kv, nil
}
ctx, cancel := context.WithTimeout(b.ctx, requestWithin)
defer cancel()
kv, err := b.js.KeyValue(ctx, bucketOf(state))
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
err = fmt.Errorf("no answer in %s: a read the bus refuses is answered by none (%w)", requestWithin, err)
}
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", bucketOf(state), err)
}
b.mu.Lock()
b.kvs[state] = kv
b.mu.Unlock()
return kv, nil
}
func (b *busReader) Keys(state string) ([]string, error) {
b.mu.Lock()
if b.settled[state] {
out := make([]string, 0, len(b.world[state]))
for k := range b.world[state] {
out = append(out, k)
}
b.mu.Unlock()
sort.Strings(out)
return out, nil
}
b.mu.Unlock()
kv, err := b.kv(state)
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(b.ctx, requestWithin)
defer cancel()
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return []string{}, nil
}
return keys, err
}
func (b *busReader) Get(state, key string) ([]byte, bool, error) {
b.mu.Lock()
if b.settled[state] {
v, ok := b.world[state][key]
b.mu.Unlock()
return v, ok, nil
}
b.mu.Unlock()
kv, err := b.kv(state)
if err != nil {
return nil, false, err
}
ctx, cancel := context.WithTimeout(b.ctx, requestWithin)
defer cancel()
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
return e.Value(), true, nil
}
// Watch keeps every value of a state from now on and hands each change after what stands to changed; it returns once
// what stands is held. The watch ending is handed to ended, and the state is read from the bus again until watched.
func (b *busReader) Watch(state string, changed func(string, []byte), ended func(error)) error {
kv, err := b.kv(state)
if err != nil {
return err
}
w, err := kv.WatchAll(b.ctx)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", bucketOf(state), err)
}
put := func(e jetstream.KeyValueEntry) {
b.mu.Lock()
defer b.mu.Unlock()
if b.world[state] == nil {
b.world[state] = map[string][]byte{}
}
if e.Operation() == jetstream.KeyValuePut {
b.world[state][e.Key()] = e.Value()
} else {
delete(b.world[state], e.Key())
}
}
lost := func(err error) {
w.Stop()
b.mu.Lock()
b.settled[state] = false
delete(b.world, state)
delete(b.kvs, state)
b.mu.Unlock()
ended(err)
}
settling, stop := context.WithTimeout(b.ctx, requestWithin)
defer stop()
settle:
for {
select {
case <-settling.Done():
w.Stop()
if b.ctx.Err() != nil {
return b.ctx.Err()
}
return fmt.Errorf("the watch of %s handed nothing in %s: a watch the bus refuses is answered by none", bucketOf(state), requestWithin)
case e, ok := <-w.Updates():
if !ok {
w.Stop()
return fmt.Errorf("the watch of %s ended before it held what stands", bucketOf(state))
}
if e == nil {
break settle
}
put(e)
}
}
b.mu.Lock()
b.settled[state] = true
b.mu.Unlock()
go func() {
for {
select {
case <-b.ctx.Done():
w.Stop()
return
case e, ok := <-w.Updates():
if !ok {
lost(errors.New("the server ended it"))
return
}
if e != nil {
put(e)
var value []byte
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
changed(e.Key(), value)
}
}
}
}()
return nil
}
// ownStore is the part's own state on its process connection: a state it did not announce, or one the bus refuses,
// fails the call, named.
type ownStore struct{ c *process.Conn }
func (s ownStore) Get(state, key string) ([]byte, uint64, bool, error) {
e, err := s.c.State(state).Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
return s.c.State(state).Put(key, json.RawMessage(value))
}
// emitting is the process connection as the part's emitter: its events, with no options.
type emitting struct{ c *process.Conn }
func (e emitting) Emit(event string, body any) error { return e.c.Emit(event, body) }
@@ -1,34 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"mesh-grants/internal/grants"
)
// creating creates a stream a manifest declares that the bus lacks (issue 1790; the operator, 2026-10-26 08:54): the
// create alone, which the account is granted by the stream's name, never an update or a delete. The one place the part
// creates on the bus.
type creating struct {
ctx context.Context
js jetstream.JetStream
}
func (c creating) CreateDeclared(cfg jetstream.StreamConfig) error {
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
defer cancel()
_, err := c.js.CreateStream(ctx, cfg)
switch {
case err == nil:
return nil
case errors.Is(err, jetstream.ErrStreamNameAlreadyInUse):
return grants.ErrHeld
case errors.Is(err, context.DeadlineExceeded):
return fmt.Errorf("no answer in %s: a create the bus refuses is answered by none (%w)", requestWithin, err)
}
return err
}
-140
View File
@@ -1,140 +0,0 @@
// Command mesh-grants is the grants part (novox/hq ADR 0547 clauses 5 to 8, issue 1945): a process on its own bus
// credential that watches placement, the assigner's wants, the indexer's manifests, the builder's builds, mesh-config's
// resolved values and the node agents' haves, and on every change composes each assignment's membership, the bus's user
// list and each manifest's buckets, logs and streams with the bus composer the controller uses (mesh-sdk buscompose).
//
// **Memberships are published for real** (the operator, 2026-10-25 22:31): since release 6 nobody publishes them, so a
// module placed since cannot serve its seat or reach a new state. Each is published on the assignments stream only when
// it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is
// created** (the operator, 2026-10-26 08:54; issue 1790), once per manifest, with the manifest's limits, on the event
// that brought the manifest; one the bus holds is never changed. **The user list, the rest of the assertions and the
// grants stay in shadow** until
// the operator's take-over, which this milestone always refuses: they are composed, compared with what the bus loaded
// and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect.
//
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
// event.
//
// **No seat in this milestone.** ADR 0547 clause 5 gives the part the seat `mesh-grants`; a `mesh-*` seat must be in
// mesh-sdk's own seat list, which the running controller's composer does not know, so the part serves module tools
// (grants_*) and claims no seat until take-over. The composer grants the membership publish by the module's name,
// mesh-grants.
//
// Started by the tool runner, with no credential of its own, it serves those tools (a process never serves, ADR 0436).
package main
import (
"context"
"errors"
"fmt"
"io"
"log"
"os"
"os/signal"
"strings"
"syscall"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-grants/internal/grants"
)
func main() {
if os.Getenv("MESH_BROKER_FILE") == "" {
if err := serveTools(); err != nil {
fmt.Fprintln(os.Stderr, "mesh-grants:", err)
os.Exit(1)
}
return
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
fmt.Fprintln(os.Stderr, "mesh-grants:", err)
os.Exit(1)
}
}
// keptStates is what the part keeps, each readable (ADR 0495 §2-§3): the view and a session read them.
func keptStates() []announce.State {
var out []announce.State
for _, s := range grants.States {
out = append(out, announce.State{Name: s, Readable: true})
}
return out
}
// run is the process: it returns when ctx ends, or when it has no credential or cannot dial the bus at all. A bucket or
// read the bus refuses never ends it.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-grants: ", log.LstdFlags)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
}
account, err := process.ReadAccount(path)
if err != nil {
return err
}
f := &grants.Follower{Logf: logger.Printf}
conn, err := process.Connect(account, nats.Name(account.Node+"."+account.Module), nats.RetryOnFailedConnect(true),
nats.ReconnectHandler(func(*nats.Conn) { f.Wake() }))
if err != nil {
return err
}
defer conn.Close()
if err := conn.Keeps(keptStates()...); err != nil {
logger.Printf("what the part keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
// The reads of other parts' state, on a connection of the same account: a read the bus refuses is said and composed
// past (busReader), never the end of the process.
opts := []nats.Option{
nats.Name(account.Node + "." + account.Module + ".reads"),
nats.UserInfo(account.User, account.Password),
nats.CustomInboxPrefix(account.InboxPrefix()),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
nats.ReconnectHandler(func(*nats.Conn) { f.Wake() }),
}
if strings.TrimSpace(account.Fingerprint) != "" {
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
}
nc, err := nats.Connect(account.URL, opts...)
if err != nil {
return fmt.Errorf("connecting to the bus: %w", err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
return err
}
bus := newBusReader(ctx, js)
f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn},
Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}}
f.Watch = bus
f.Asked = func(wake func()) error {
return conn.State(grants.StateUserList).Watch(grants.KeyAsked, func(c stdio.StateChange) error {
if !c.Current {
wake()
}
return nil
})
}
f.Hear = func(handle func(string, []byte)) error {
for _, event := range grants.Consumes {
event := event
if _, err := nc.Subscribe(grants.EventSubject(event), func(m *nats.Msg) { handle(event, m.Data) }); err != nil {
return fmt.Errorf("hearing %s: %w", event, err)
}
}
return nil
}
logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; the user list, grants and any change to a held stream in shadow", account.User)
f.Run(ctx)
return nil
}
@@ -1,55 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strconv"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/membership"
)
// memberships publishes each membership on the assignments stream, last per subject, over the one last read there
// (ExpectLastSequencePerSubject, so a membership another writer published meanwhile is not overwritten, issues 930 and
// 934), and reads the last one by a direct get of its subject. The one place the part publishes on the bus.
type memberships struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
}
func (m memberships) LastMembership(node, module string) ([]byte, uint64, bool, error) {
ctx, cancel := context.WithTimeout(m.ctx, requestWithin)
defer cancel()
msg, err := m.nc.RequestWithContext(ctx, membership.DirectGetSubject(node, module), nil)
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
err = fmt.Errorf("no answer in %s: a read the bus refuses is answered by none (%w)", requestWithin, err)
}
return nil, 0, false, err
}
switch status := msg.Header.Get("Status"); status {
case "":
case "404":
return nil, 0, false, nil
default:
return nil, 0, false, fmt.Errorf("the assignments stream answered %s %s", status, msg.Header.Get("Description"))
}
seq, err := strconv.ParseUint(msg.Header.Get("Nats-Sequence"), 10, 64)
if err != nil {
return nil, 0, false, fmt.Errorf("the membership of %s on %s carries no sequence", module, node)
}
return msg.Data, seq, true, nil
}
func (m memberships) PublishMembership(node, module string, body []byte, after uint64) error {
ctx, cancel := context.WithTimeout(m.ctx, requestWithin)
defer cancel()
if _, err := m.js.Publish(ctx, membership.Subject(node, module), body, jetstream.WithExpectLastSequencePerSubject(after)); err != nil {
return fmt.Errorf("the membership of %s on %s after sequence %d: %w", module, node, after, err)
}
return nil
}
@@ -1,121 +0,0 @@
package main
import (
"encoding/json"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-grants/internal/grants"
)
// The part's tools, served through the tool runner from its bundle (ADR 0436: a process never serves). They read and
// write through the runtime as the module: its own states, the reads its manifest names, the bus module's read tools.
// runtimeReader reads another part's state through the runtime, `<module>.<state>`.
type runtimeReader struct{}
func (runtimeReader) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
func (runtimeReader) Get(state, key string) ([]byte, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
// runtimeStore is the part's own state through the runtime.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (runtimeStore) Put(state, key string, value []byte) (uint64, error) {
return stdio.State(state).Put(key, json.RawMessage(value))
}
// runtimeBus calls tools and says events through the runtime.
type runtimeBus struct{}
func (runtimeBus) Call(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }
func (runtimeBus) Emit(event string, body any) error { return stdio.Emit(event, body) }
func schema(properties map[string]string, required ...string) map[string]any {
props := map[string]any{}
for name, description := range properties {
props[name] = map[string]any{"type": "string", "description": description}
}
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
out["required"] = required
}
return out
}
// toolsOf are the part's tools over a part (grants.Tools names them).
func toolsOf(p *grants.Part) []stdio.Tool {
return []stdio.Tool{
{Name: grants.ToolGrants, Description: "What the grants part holds of grants (novox/hq ADR 0547 clause 6): in this milestone it " +
"writes none (grants.<module>.<fp> from the third milestone, clause 5), and shows beside it the grants the controller " +
"wrote (mesh-controller_grants), all or one module's with the accounts each names. Read-only.",
Input: schema(map[string]string{"module": "one module"}),
Run: func(a map[string]any) (any, error) { return p.Grants(grants.Answer(a, "module")) }},
{Name: grants.ToolMembership, Description: "One module's membership on one node (ADR 0547 clause 6): as composed now from the " +
"bus's records with the bus composer, and what the part's process published of it at its last pass (published, " +
"unchanged, or why not: a shadow difference names what publishing would lose; verbs the seat catalogue retired are " +
"dropped and named), with what the composition cannot read. With neither node nor module: every membership the last " +
"pass held back, each with what it would lose. Read-only.",
Input: schema(map[string]string{"node": "the machine (with module)", "module": "the module (with node)"}),
Run: func(a map[string]any) (any, error) {
return p.MembershipOf(grants.Answer(a, "node"), grants.Answer(a, "module"))
}},
{Name: grants.ToolUserList, Description: "The bus's user list in shadow (ADR 0547 clauses 6 and 8): the list the grants part " +
"composed (digest, revision, users), what the bus node says it loaded (source, revision, digest, users, error), and how " +
"they differ: users added, users dropped (and of those, gone from placement), each user's grant differences, the " +
"assignments left out, every input the bus does not carry, named; and per user whether the vault holds its hash " +
"(mesh-vault.bus-accounts, ADR 0547 clause 9) and how its line would differ from the loaded one (same, adds, left-out " +
"for want of a credential, loses-login), never a hash. Read-only.",
Run: func(map[string]any) (any, error) { return p.UserList() }},
{Name: grants.ToolComposeNow, Description: "Compose again now (ADR 0547 clause 6), as the part does on every change it hears: asks " +
"the part's process (which alone holds the grant to publish a membership) to compose the memberships, the user list and " +
"the assertions, publish each membership that differs from the one last published, and keep what changed in its own " +
"state (one revision when an input changed, none when not), saying user-list-changed and membership-changed. Answers " +
"the ask and the last pass; grants_user_list shows the pass that answers it. A membership is published only when it is " +
"the first or only adds to the last one; any loss stays a shadow difference, named. The user list, assertions and " +
"grants stay in shadow. It cannot ask the controller to write a module's grant again: the controller serves no such " +
"verb, and the answer says so.",
Run: func(a map[string]any) (any, error) { return p.AskCompose(stdio.CallerOf(a)) }},
{Name: grants.ToolAssert, Description: "One module's buckets, logs and streams (ADR 0547 clause 6), composed from its manifest at " +
"the indexer's trunk with their limits, beside what the bus holds (through the bus module's nats_streams). When the " +
"bus lacks one, asks the part's process (which alone holds the grant) to create it now with the manifest's limits, " +
"as it does by itself when a manifest is registered (issue 1790), and says asserted; one the bus holds is never " +
"changed, a difference is only said. Answers the comparison and the ask; call again to see what the bus holds.",
Input: schema(map[string]string{"module": "the module"}, "module"),
Run: func(a map[string]any) (any, error) { return p.AssertOne(grants.Answer(a, "module")) }},
{Name: grants.ToolTakeOver, Description: "The operator's switch to the grants part's list (ADR 0547 clause 8), refused without the " +
"operator's credential (a person's account), refused while any user of the shadow list differs from the loaded list (but " +
"those placement no longer names), naming them; in this milestone always refused, since the enrolment users and the " +
"persons cannot be composed. Every ask is recorded in the part's state takeover.",
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
Run: func(a map[string]any) (any, error) {
return nil, p.TakeOver(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolHandBack, Description: "The operator's switch back to the controller's list (ADR 0547 clause 8), refused without " +
"the operator's credential; in shadow there is nothing to hand back, and it says so. Every ask is recorded.",
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
Run: func(a map[string]any) (any, error) {
return nil, p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
}
}
// serveTools serves the tools to the runtime until it stops.
func serveTools() error {
p := &grants.Part{Read: runtimeReader{}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}}
return stdio.Serve("", toolsOf(p))
}
-16
View File
@@ -1,16 +0,0 @@
module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
-26
View File
@@ -1,26 +0,0 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896 h1:5rHfru5NJ7SK20QOt8YUJSglcn1gsjtLc9AGmWYGnRQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
File diff suppressed because it is too large Load Diff
@@ -1,245 +0,0 @@
package grants
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Item is one bucket, log or stream a manifest declares, as composed, beside what the bus holds of it.
type Item struct {
Kind string `json:"kind"`
Stream string `json:"stream"`
// Composed is what an assertion would ask for; Held what the bus holds, absent when it holds no such stream.
Composed Limits `json:"composed"`
Held *Limits `json:"held,omitempty"`
// Differences says how they differ, in words; empty when they agree in what can be read from here.
Differences []string `json:"differences,omitempty"`
// Created says what became of creating it when the bus lacked it: "created", or why not (issue 1790). Empty when
// the bus holds it or no creation was asked here.
Created string `json:"created,omitempty"`
}
// Limits are the facts of a stream an assertion sets and the bus module's nats_streams reads back. Its size cap is not
// read back by that tool, so it is composed and not compared.
type Limits struct {
Subjects []string `json:"subjects"`
Retention string `json:"retention"`
MaxAgeHours int `json:"max_age_hours,omitempty"`
MaxPerSubject int `json:"max_per_subject,omitempty"`
MaxBytes int64 `json:"max_bytes,omitempty"`
}
// Assertion is one module's declared buckets, logs and streams, composed and compared. Asserted says this pass created
// one the bus lacked (the operator, 2026-10-26 08:54); one the bus holds is never changed.
type Assertion struct {
Module string `json:"module"`
Manifest string `json:"manifest"`
Items []Item `json:"items"`
Asserted bool `json:"asserted"`
Says string `json:"says"`
}
// ShadowAsserts is what an assertion says where nothing may be created: in the tools, which ask the part's process.
const ShadowAsserts = "in shadow: composed and compared, nothing asserted (ADR 0547 clause 5 asserts from take-over)"
// CreatesMissing is what an assertion of the part's process says (the operator, 2026-10-26 08:54; issue 1790).
const CreatesMissing = "a bucket, log or stream the manifest declares and the bus lacks is created with the manifest's limits, once per " +
"manifest; one the bus holds is never changed or deleted, a difference is only said (ADR 0547 clause 5 asserts the rest from take-over)"
// Creator creates one stream the bus does not hold, with the configuration given, and never updates or deletes one: the
// part's process alone, whose account is granted the create of each declared stream by name (the operator, 2026-10-26
// 08:54). A stream the bus holds already answers ErrHeld.
type Creator interface {
CreateDeclared(cfg jetstream.StreamConfig) error
}
// ErrHeld is a create refused because the bus holds the stream already (made meanwhile): left as it is.
var ErrHeld = errors.New("the bus holds it already, so it is left as it is")
// Created words of an item.
const (
ItemCreated = "created"
ItemNotCreated = "not created: "
)
// Configs are the streams a manifest declares, as the controller asserts them at its start (mesh-sdk buscompose), by
// stream name. Pure.
func Configs(m catalogue.Manifest) map[string]jetstream.StreamConfig {
out := map[string]jetstream.StreamConfig{}
for _, b := range bucketsOf(m) {
c := buscompose.BucketStreamConfig(b)
out[c.Name] = c
}
for _, l := range logsOf(m) {
c := l.Config()
out[c.Name] = c
}
for _, s := range streamsOf(m) {
c := buscompose.ModuleStreamConfig(s)
out[c.Name] = c
}
return out
}
// creations remembers each stream created, by the manifest digest it was created at: created once per manifest, so a
// stream deleted since is not made again until a new manifest or a hand assert (grants_assert).
type creations struct {
mu sync.Mutex
made map[string]string
}
func (c *creations) madeAt(stream string) string {
c.mu.Lock()
defer c.mu.Unlock()
return c.made[stream]
}
func (c *creations) keep(stream, digest string) {
c.mu.Lock()
defer c.mu.Unlock()
if c.made == nil {
c.made = map[string]string{}
}
c.made[stream] = digest
}
// createMissing creates each item of a the bus lacks, with the manifest's limits, and says what became of it. It never
// touches an item the bus holds, and does nothing when what the bus holds was not read. force asks again a stream
// created at this manifest and gone since (a hand assert). It answers the streams created.
func createMissing(c Creator, done *creations, a *Assertion, m catalogue.Manifest, force bool) []string {
cfgs := Configs(m)
var made []string
a.Says = CreatesMissing
for i := range a.Items {
it := &a.Items[i]
if it.Held != nil || !slices.Contains(it.Differences, "the bus holds no stream "+it.Stream) {
continue
}
if !force && done.madeAt(it.Stream) == a.Manifest {
it.Created = ItemNotCreated + "created at this manifest and gone since; grants_assert creates it again"
continue
}
cfg, ok := cfgs[it.Stream]
if !ok {
it.Created = ItemNotCreated + "no configuration composed for it"
continue
}
switch err := c.CreateDeclared(cfg); {
case err == nil:
it.Created = ItemCreated
done.keep(it.Stream, a.Manifest)
made = append(made, it.Kind+" "+it.Stream)
case errors.Is(err, ErrHeld):
it.Created = ItemNotCreated + ErrHeld.Error()
default:
it.Created = ItemNotCreated + err.Error() + " (tried again at the next event)"
}
}
if len(made) > 0 {
a.Asserted = true
}
return made
}
// notCreated is each item of a creation refused, with why, as `<stream>: <why>`.
func notCreated(a Assertion) []string {
var out []string
for _, it := range a.Items {
if why, ok := strings.CutPrefix(it.Created, ItemNotCreated); ok && it.Created != "" {
out = append(out, it.Stream+": "+why)
}
}
return out
}
// Composed is a module's buckets, logs and streams as an assertion would ask for them. Pure.
func Composed(m catalogue.Manifest) []Item {
var out []Item
for _, b := range bucketsOf(m) {
per := b.History
if per < 1 {
per = 1
}
out = append(out, Item{Kind: "bucket", Stream: "KV_" + b.Bucket(), Composed: Limits{
Subjects: []string{"$KV." + b.Bucket() + ".>"}, Retention: string(buscompose.RetentionLimits),
MaxAgeHours: b.TTLSeconds / 3600, MaxPerSubject: per}})
}
for _, l := range logsOf(m) {
c := l.Config()
out = append(out, Item{Kind: "log", Stream: l.Stream(), Composed: Limits{Subjects: set(c.Subjects),
Retention: string(buscompose.RetentionLimits), MaxAgeHours: int(c.MaxAge.Hours()), MaxBytes: c.MaxBytes}})
}
for _, s := range streamsOf(m) {
st := s.Stream()
out = append(out, Item{Kind: "stream", Stream: st.Name, Composed: Limits{Subjects: set(st.Subjects),
Retention: string(st.Retention), MaxAgeHours: st.MaxAge / 3600, MaxBytes: st.MaxBytes}})
}
return out
}
// HeldStreams reads every stream the bus holds through its module's nats_streams, by name.
func HeldStreams(c Caller) (map[string]Limits, error) {
raw, err := c.Call(CallStreams, map[string]any{})
if err != nil {
return nil, fmt.Errorf("%s: %w", CallStreams, err)
}
var got struct {
Detail []struct {
Name string `json:"name"`
Subjects []string `json:"subjects"`
Retention string `json:"retention"`
MaxAgeHours int `json:"max_age_hours"`
MaxPerSubject int `json:"max_per_subject"`
} `json:"detail"`
}
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
return nil, fmt.Errorf("%s answered no streams: %w", CallStreams, err)
}
out := map[string]Limits{}
for _, s := range got.Detail {
out[s.Name] = Limits{Subjects: set(s.Subjects), Retention: s.Retention, MaxAgeHours: s.MaxAgeHours, MaxPerSubject: s.MaxPerSubject}
}
return out, nil
}
// Assert composes a module's items and compares each with what the bus holds; held nil is not read. It asserts nothing.
func Assert(module, digest string, m catalogue.Manifest, held map[string]Limits, heldErr error) Assertion {
a := Assertion{Module: module, Manifest: digest, Items: Composed(m), Says: ShadowAsserts}
for i := range a.Items {
it := &a.Items[i]
if heldErr != nil {
it.Differences = []string{"what the bus holds was not read: " + heldErr.Error()}
continue
}
h, ok := held[it.Stream]
if !ok {
it.Differences = []string{"the bus holds no stream " + it.Stream}
continue
}
it.Held = &h
if !slices.Equal(h.Subjects, it.Composed.Subjects) {
it.Differences = append(it.Differences, fmt.Sprintf("subjects %v held, %v composed", h.Subjects, it.Composed.Subjects))
}
if h.Retention != it.Composed.Retention {
it.Differences = append(it.Differences, fmt.Sprintf("retention %s held, %s composed", h.Retention, it.Composed.Retention))
}
if h.MaxAgeHours != it.Composed.MaxAgeHours {
it.Differences = append(it.Differences, fmt.Sprintf("age %d h held, %d h composed", h.MaxAgeHours, it.Composed.MaxAgeHours))
}
if it.Kind == "bucket" && max(h.MaxPerSubject, 1) != it.Composed.MaxPerSubject {
it.Differences = append(it.Differences, fmt.Sprintf("history %d held, %d composed", h.MaxPerSubject, it.Composed.MaxPerSubject))
}
}
sort.SliceStable(a.Items, func(i, j int) bool { return a.Items[i].Stream < a.Items[j].Stream })
return a
}
@@ -1,240 +0,0 @@
package grants
import (
"context"
"encoding/json"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Watcher watches another part's state: changed hears each change after what stands, ended hears the watch end. A watch
// refused (a read not granted yet, a bucket not there) is an error, and is tried again at the next wake.
type Watcher interface {
Watch(state string, changed func(key string, value []byte), ended func(error)) error
}
// Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts}
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
type Follower struct {
Part *Part
Watch Watcher
Logf func(string, ...any)
// Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none.
Asked func(wake func()) error
// Hear hears the controller seat's condition events (Consumes) and hands each, by event name, with its body, to
// handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: a machine's
// root-free word changed.
Hear func(handle func(event string, body []byte)) error
wake chan struct{}
once sync.Once
mu sync.Mutex
asked bool
heard_ bool
rootSaid string
watched map[string]bool
refused map[string]string
have map[string]string
// held and notAsserted are what the last pass held back and could not create, said again only when they change.
held string
notAsserted string
refusedMans string
}
func (f *Follower) init() {
f.once.Do(func() {
f.wake = make(chan struct{}, 1)
f.watched, f.refused, f.have = map[string]bool{}, map[string]string{}, map[string]string{}
})
}
// Wake asks for one more composition: after a change, and after a reconnect.
func (f *Follower) Wake() {
f.init()
select {
case f.wake <- struct{}{}:
default:
}
}
// Refused is every watch refused at the last try, with why.
func (f *Follower) Refused() map[string]string {
f.init()
f.mu.Lock()
defer f.mu.Unlock()
out := map[string]string{}
for k, v := range f.refused {
out[k] = v
}
return out
}
func (f *Follower) logf(format string, args ...any) {
if f.Logf != nil {
f.Logf(format, args...)
}
}
// heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the
// bus module's loaded list moving.
func (f *Follower) heard(state, key string, value []byte) bool {
if state != ReadHave {
return true
}
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil {
return false
}
f.mu.Lock()
defer f.mu.Unlock()
if isNode {
if _, seen := f.have[node]; seen {
return false
}
f.have[node] = ""
return true
}
if module != BusModule {
return false
}
var h struct {
UserList json.RawMessage `json:"user-list"`
}
_ = json.Unmarshal(value, &h)
was, seen := f.have[key]
f.have[key] = string(h.UserList)
return !seen || was != string(h.UserList)
}
// open opens every watch not open yet; one refused is said and left for the next wake.
func (f *Follower) open() {
if f.Asked != nil && !f.asked {
if err := f.Asked(f.Wake); err != nil {
f.logf("a tool's ask to compose now cannot be heard yet (%v): tried again at the next change", err)
} else {
f.asked = true
}
}
if f.Hear != nil && !f.heard_ {
err := f.Hear(func(event string, body []byte) {
if IsRootNotFree(body) {
f.Wake()
}
})
if err != nil {
f.logf("the controller's condition events cannot be heard yet (%v): tried again at the next change", err)
} else {
f.heard_ = true
}
}
for _, state := range Followed {
f.mu.Lock()
open := f.watched[state]
f.mu.Unlock()
if open {
continue
}
state := state
err := f.Watch.Watch(state, func(key string, value []byte) {
if f.heard(state, key, value) {
f.Wake()
}
}, func(err error) {
f.mu.Lock()
f.watched[state] = false
f.mu.Unlock()
f.logf("the watch of %s ended (%v): opened again at the next change", state, err)
f.Wake()
})
f.mu.Lock()
if err != nil {
f.refused[state] = err.Error()
f.logf("%s cannot be watched yet (%v): composed without it, and tried again at the next change or reconnect", state, err)
} else {
f.watched[state] = true
delete(f.refused, state)
}
f.mu.Unlock()
}
}
// said logs every membership the pass held back, each with what it would lose (issue 1945), and every declared stream
// the pass could not create, each with why (issue 1790), when the set changes from the last pass's; and every stream it
// created, one line per module.
func (f *Follower) said(res Result) {
if refused := strings.Join(res.Refused, "\n"); refused != f.refusedMans {
f.refusedMans = refused
if len(res.Refused) == 0 {
f.logf("every manifest the indexer holds is read")
} else {
f.logf("%d manifests cannot be read, so their modules count as placed nowhere (grants_membership with no node and module lists them):",
len(res.Refused))
for _, r := range res.Refused {
f.logf("refused %s", r)
}
}
}
if held := strings.Join(res.NotPublished, "\n"); held != f.held {
f.held = held
if len(res.NotPublished) == 0 {
f.logf("no membership held back")
} else {
f.logf("%d memberships not published, tried again at the next event (grants_membership with no node and module lists them):",
len(res.NotPublished))
for _, n := range res.NotPublished {
f.logf("held %s", n)
}
}
}
if said := strings.Join(res.RootNotFree, "\n"); said != f.rootSaid {
f.rootSaid = said
if len(res.RootNotFree) == 0 {
f.logf("every machine a composition needs a root-free word for is free")
}
for _, n := range res.RootNotFree {
f.logf("root-free: %s (a verified sender is composed as from no root-free machine there)", n)
}
}
for _, a := range res.Asserted {
f.logf("asserted %s", a)
}
if not := strings.Join(res.NotAsserted, "\n"); not != f.notAsserted {
f.notAsserted = not
for _, n := range res.NotAsserted {
f.logf("not asserted %s", n)
}
}
}
// Run composes at start and after every change until ctx ends. It returns only when ctx ends.
func (f *Follower) Run(ctx context.Context) {
f.init()
f.Wake()
for {
select {
case <-ctx.Done():
return
case <-f.wake:
f.open()
res := f.Part.Pass()
for _, e := range res.Errors {
f.logf("%s", e)
}
if len(res.Published) > 0 {
f.logf("published %d memberships: %v", len(res.Published), res.Published)
}
f.said(res)
if res.Written {
f.logf("the shadow list moved to revision %d: %d users, %d differing from the loaded list",
res.Revision, res.UserList.Users, len(res.UserList.Compare.Differing()))
}
}
}
}
@@ -1,363 +0,0 @@
// Package grants is the grants part in shadow (novox/hq ADR 0547 clauses 5 to 8, issue 1945): it composes the bus's user
// list, each module's membership and each manifest's buckets, logs and streams from what the bus holds, with the bus
// composer the controller uses (mesh-sdk buscompose), and compares them with what the bus loaded and holds. Beyond its own
// state it takes effect in two things alone, each by a grant the operator allowed: it publishes memberships (2026-10-25
// 22:31), and creates a bucket, log or stream a manifest declares that the bus lacks (2026-10-26 08:54, issue 1790). It
// changes no stream the bus holds, writes no grant and no user list the bus node reads, until the operator's take-over,
// which this milestone always refuses.
package grants
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The states of other parts the grants part reads, as its manifest's reads name them (`<module>.<state>`).
const (
ReadAssignments = "mesh-controller.assignments"
ReadControllerGrant = "mesh-controller.grants"
ReadModules = "mesh-delivery-indexer.modules"
ReadManifests = "mesh-delivery-indexer.manifests"
ReadHave = "node-engine.have"
ReadResolved = "mesh-config.resolved"
ReadWants = "mesh-delivery-assigner.wants"
ReadBuilds = "mesh-delivery-builder.builds"
// ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9).
ReadAccounts = "mesh-vault.bus-accounts"
)
// Reads is every state the part reads, in the manifest's order.
var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
ReadAccounts}
// The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
const (
CallUsers = "nats.nats_users"
CallUserCan = "nats.nats_user_can"
CallStreams = "nats.nats_streams"
)
// Invokes is every tool the part calls.
var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree}
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
// an error the part names as a gap and composes past: it never stops for one.
type Reader interface {
Keys(state string) ([]string, error)
Get(state, key string) ([]byte, bool, error)
}
// Caller calls another module's tool, `<module>.<tool>`, and answers its result.
type Caller interface {
Call(key string, body any) (json.RawMessage, error)
}
// Gap is an input the shadow cannot read from the bus, named with why: never guessed.
type Gap struct {
Input string `json:"input"`
Why string `json:"why"`
}
// Standing gaps: what the controller composes from its own store, which no bus value carries in this milestone.
var standingGaps = []Gap{
{Input: "enrolment tokens", Why: "the live enrolment tokens are in the controller's store alone: no enrolment user is composed"},
{Input: "people", Why: "the people and the tools they may invoke are in the controller's store alone: no person user is composed"},
{Input: "seat holdings", Why: "who holds each seat held once for the mesh is in the controller's store alone: a mesh seat claimed on two machines is composed as held on both"},
{Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"},
}
// assignment is a key of the controller's assignments as composing needs it (mesh-controller internal/chain Assignment):
// a value that names no time assigned (a module pulled in), has ended, or waits in a state, is no assignment.
type assignment struct {
Node string `json:"node"`
Module string `json:"module"`
Assigned time.Time `json:"assigned,omitzero"`
Ended time.Time `json:"ended,omitzero"`
State string `json:"state,omitempty"`
}
func (a assignment) stands() bool { return !a.Assigned.IsZero() && a.Ended.IsZero() && a.State == "" }
// resolved is a key of mesh-config's resolved values, `<node>.<module>.<key>`.
type resolved struct {
Node string `json:"node"`
Module string `json:"module"`
Key string `json:"key"`
Value json.RawMessage `json:"value,omitempty"`
}
// Inputs is what the shadow composes from, read from the bus.
type Inputs struct {
// Nodes is every machine seen: in a node agent's have or a standing assignment.
Nodes []string
// Assigned is each machine's standing assignments, sorted.
Assigned map[string][]string
// Manifests is each module's manifest at the indexer's trunk, by module.
Manifests map[string]catalogue.Manifest
// Digests is each module's manifest digest, by module.
Digests map[string]string
// Running is the manifest a machine is wanted to run, by `<node>.<module>`: the one the builder built at the
// fingerprint the assigner's want names. An assignment with none is composed from the trunk's manifest.
Running map[string]catalogue.Manifest
// Values is each resolved config value, by `<node>/<module>/<key>`.
Values map[string]any
// BusNode is the machine running the bus's module, and Loaded what its node agent says it loaded.
BusNode string
Loaded *delivery.UserListWritten
// Hashes are the vault's hashes of the bus accounts, by bus user (ReadAccounts); kept in memory, never written.
Hashes map[string]HeldHash
// Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or
// not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945).
Refused []Gap
Gaps []Gap
}
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said or written by
// the part in shadow.
type HeldHash struct {
Hash string `json:"hash"`
Generation int `json:"generation,omitempty"`
At time.Time `json:"at"`
Adopted bool `json:"adopted,omitempty"`
}
// readHashes reads the vault's bus-accounts; a refusal is a gap, and every user then has no hash held.
func readHashes(r Reader, gaps *[]Gap) map[string]HeldHash {
out := map[string]HeldHash{}
for user, raw := range each(r, ReadAccounts, gaps) {
var h HeldHash
if json.Unmarshal(raw, &h) != nil || h.Hash == "" {
*gaps = append(*gaps, Gap{Input: ReadAccounts + " " + user, Why: "holds no hash"})
continue
}
out[user] = h
}
return out
}
// BusModule is the bus's module: it holds mesh-broker.
const BusModule = "nats"
// Manifests caches manifests by digest: a digest names one manifest for ever.
type Manifests struct {
mu sync.Mutex
parsed map[string]catalogue.Manifest
}
func (c *Manifests) get(r Reader, digest string) (catalogue.Manifest, error) {
// The lock guards the map alone, never a read on the bus: a tool never waits on a pass's round trip.
c.mu.Lock()
m, ok := c.parsed[digest]
c.mu.Unlock()
if ok {
return m, nil
}
raw, found, err := r.Get(ReadManifests, digest)
if err != nil {
return catalogue.Manifest{}, err
}
if !found {
return catalogue.Manifest{}, fmt.Errorf("the indexer holds no manifest %s", digest)
}
v, err := delivery.DecodeManifest(raw)
if err != nil {
return catalogue.Manifest{}, err
}
// Read for its inputs, as every delivery part reads a manifest (issue 1751): the merge gate judged its config values,
// and the part reads its seats, buckets, logs and streams, never a value; so a value falling back to a predefined key
// is taken as it declares itself, where ParseManifest refused it with no list and placed its module nowhere (issue 1945).
m, err = catalogue.ParseManifestForInputs([]byte(v.JSON))
if err != nil {
return catalogue.Manifest{}, fmt.Errorf("the manifest %s is refused by mesh-sdk's reader: %w", digest, err)
}
c.mu.Lock()
defer c.mu.Unlock()
if c.parsed == nil {
c.parsed = map[string]catalogue.Manifest{}
}
c.parsed[digest] = m
return m, nil
}
// each reads every current value of a state; a refusal is a gap and an empty map.
func each(r Reader, state string, gaps *[]Gap) map[string][]byte {
keys, err := r.Keys(state)
if err != nil {
*gaps = append(*gaps, Gap{Input: state, Why: "refused or absent: " + err.Error()})
return map[string][]byte{}
}
out := map[string][]byte{}
for _, k := range keys {
v, found, err := r.Get(state, k)
if err != nil {
*gaps = append(*gaps, Gap{Input: state + " " + k, Why: err.Error()})
continue
}
if found {
out[k] = v
}
}
return out
}
// ReadInputs reads what the shadow composes from. It never fails: what it cannot read is a gap.
func ReadInputs(r Reader, cache *Manifests) Inputs {
in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{},
Running: map[string]catalogue.Manifest{},
Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
nodes := map[string]bool{}
for _, raw := range each(r, ReadAssignments, &in.Gaps) {
var a assignment
if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" {
continue
}
nodes[a.Node] = true
in.Assigned[a.Node] = append(in.Assigned[a.Node], a.Module)
if a.Module == BusModule {
in.BusNode = a.Node
}
}
haveKeys, err := r.Keys(ReadHave)
if err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error()})
}
for _, k := range haveKeys {
if node, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode {
nodes[node] = true
}
}
in.Gaps = append(in.Gaps, Gap{Input: "machines", Why: "read from the node agents' haves and the assignments: a machine with neither is not seen"})
for n := range nodes {
in.Nodes = append(in.Nodes, n)
}
sort.Strings(in.Nodes)
if in.BusNode != "" {
if raw, found, err := r.Get(ReadHave, in.BusNode+"."+BusModule); err != nil {
in.Gaps = append(in.Gaps, Gap{Input: "the loaded list's have", Why: err.Error()})
} else if found {
if h, err := delivery.DecodeHave(raw); err == nil {
in.Loaded = h.UserList
}
}
}
wanted := map[string]bool{}
for node := range in.Assigned {
sort.Strings(in.Assigned[node])
for _, m := range in.Assigned[node] {
wanted[m] = true
}
}
modules := make([]string, 0, len(wanted))
for m := range wanted {
modules = append(modules, m)
}
sort.Strings(modules)
// The seats every module declares, across the whole catalogue: read every module the indexer holds, not only the
// assigned ones, since a seat is declared by one module and held by another.
indexed, err := r.Keys(ReadModules)
if err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadModules, Why: "refused or absent: " + err.Error()})
}
for _, module := range indexed {
raw, found, err := r.Get(ReadModules, module)
if err != nil || !found {
continue
}
v, err := delivery.DecodeModule(raw)
if err == nil && v.Manifest == "" {
err = fmt.Errorf("the indexer's value names no manifest digest")
}
var m catalogue.Manifest
if err == nil {
m, err = cache.get(r, v.Manifest)
}
if err != nil {
// Every manifest, assigned or not: one refused declares no seat, so its holders compose as placed nowhere.
g := Gap{Input: "manifest of " + module, Why: err.Error()}
in.Refused = append(in.Refused, g)
in.Gaps = append(in.Gaps, g)
continue
}
in.Manifests[module] = m
in.Digests[module] = v.Manifest
}
for _, module := range modules {
if _, ok := in.Manifests[module]; !ok && !gapNamed(in.Gaps, "manifest of "+module) {
in.Gaps = append(in.Gaps, Gap{Input: "manifest of " + module,
Why: "assigned, and the indexer holds no manifest of it at the trunk (a module registered from outside the indexed repositories): composed with no authority"})
}
}
// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
var fromTrunk []string
for key, raw := range each(r, ReadWants, &in.Gaps) {
node, module, err := delivery.ParseWantKey(key)
if err != nil || !slices.Contains(in.Assigned[node], module) {
continue
}
w, err := delivery.DecodeWant(raw)
if err != nil {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
if w.Removed || w.FP == "" {
continue
}
bk, err := delivery.BuilderBuildKey(module, w.FP)
if err != nil {
continue
}
braw, found, err := r.Get(ReadBuilds, bk)
if err != nil || !found {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
b, err := delivery.DecodeBuilderBuild(braw)
if err != nil || b.Manifest == "" {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
if m, err := cache.get(r, b.Manifest); err == nil {
in.Running[node+"."+module] = m
} else {
fromTrunk = append(fromTrunk, node+"."+module)
}
}
if len(fromTrunk) > 0 {
sort.Strings(fromTrunk)
in.Gaps = append(in.Gaps, Gap{Input: "wanted manifests", Why: "no build record names the manifest at the wanted fingerprint, so the trunk's is used: " + strings.Join(fromTrunk, ", ")})
}
in.Hashes = readHashes(r, &in.Gaps)
for _, raw := range each(r, ReadResolved, &in.Gaps) {
var v resolved
if json.Unmarshal(raw, &v) != nil || v.Node == "" || v.Module == "" || v.Key == "" || len(v.Value) == 0 {
continue
}
var value any
if json.Unmarshal(v.Value, &value) == nil {
in.Values[v.Node+"/"+v.Module+"/"+v.Key] = value
}
}
sortGaps(in.Gaps)
return in
}
func gapNamed(gaps []Gap, input string) bool {
for _, g := range gaps {
if g.Input == input {
return true
}
}
return false
}
func sortGaps(g []Gap) {
sort.SliceStable(g, func(i, j int) bool { return strings.Compare(g[i].Input, g[j].Input) < 0 })
}
@@ -1,256 +0,0 @@
package grants
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"reflect"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/membership"
)
// Memberships are published for real from this milestone (the operator, 2026-10-25 22:31): since release 6 nobody
// publishes them, so a module placed since cannot serve its seat or reach a new state. The user list, the assertions and
// the grants stay in shadow until take-over.
// MembershipGaps are what a membership carries that the part cannot compose from the bus.
var MembershipGaps = []Gap{
{Input: "membership: private network and received contributions", Why: "kept in each node's composition, which only that node reads: carried over from the membership last published, none for a first one"},
{Input: "membership: root-free machines", Why: "asked of the controller's root-free verb at each pass and kept for that pass alone; a machine it does not call free (an error included) is composed as not free, and the pass says why"},
}
// Publisher publishes a membership on the assignments stream (membership.Subject), last per subject, and reads the one
// last published there. A refusal (the grant not yet on the bus) is an error the part says and tries again at the next
// event.
type Publisher interface {
LastMembership(node, module string) (body []byte, seq uint64, found bool, err error)
PublishMembership(node, module string, body []byte, after uint64) error
}
// Membership is one module's membership on one node, as composed now, and what became of its publishing.
type Membership struct {
Node string `json:"node"`
Module string `json:"module"`
Digest string `json:"digest"`
Membership membership.Membership `json:"membership"`
// Published says what became of it at the last pass: "published", "unchanged" (equal to the one last published),
// or why it was not.
Published string `json:"published"`
// Retired are the seat verbs the last membership served that mesh-sdk's seat catalogue no longer has, `<seat>
// <verb>`: dropped on purpose, so their loss does not hold the membership back (issue 1945).
Retired []string `json:"retired,omitempty"`
}
// Memberships composes every assignment's membership from the records (buscompose.MembershipFor, as the controller's
// parked composition pass, mesh-controller 003eee8a, issued them), by `<node>.<module>`. Pure.
func Memberships(r buscompose.Records) map[string]Membership {
where := buscompose.PlacementsOf(r, r.Interchangeable)
out := map[string]Membership{}
for node, declared := range r.Assigned {
for _, d := range declared {
m := buscompose.MembershipFor(node, d, where)
out[node+"."+d.Module] = Membership{Node: node, Module: d.Module, Digest: digestOf(m), Membership: m}
}
}
return out
}
func digestOf(m membership.Membership) string {
raw, _ := json.Marshal(m)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// Published words of a membership's last pass.
const (
MembershipPublished = "published"
MembershipUnchanged = "unchanged"
)
// publish publishes m when it differs from the membership last published for its assignment, carrying that one's private
// network and received contributions; it answers m as published (or why not) and whether it was published now.
func publish(p Publisher, m Membership) (Membership, bool) {
if p == nil {
m.Published = "not published: no publisher here (only the part's process publishes)"
return m, false
}
was, seq, found, err := p.LastMembership(m.Node, m.Module)
if err != nil {
m.Published = "not published: the membership last published cannot be read: " + err.Error()
return m, false
}
if found {
var last membership.Membership
if json.Unmarshal(was, &last) == nil {
m.Membership.Mesh, m.Membership.Receives = last.Mesh, last.Receives
m.Digest = digestOf(m.Membership)
}
}
body, err := json.Marshal(m.Membership)
if err != nil {
m.Published = "not published: " + err.Error()
return m, false
}
if found && sameMembership(was, body) {
m.Published = MembershipUnchanged
return m, false
}
// Only a first membership, or one that only adds to the last, is published (the lead's decision, review of 1ff5049):
// composed with no root-free judgement, a kind's verified-sender would be dropped, and the operator's approvals by
// a channel would no longer be honoured. Any loss stays a shadow difference, named.
if found {
kept, retired := WithoutRetired(was)
m.Retired = retired
if lost := Losses(kept, body); len(lost) > 0 {
m.Published = ShadowDifference + strings.Join(lost, "; ")
return m, false
}
}
if err := p.PublishMembership(m.Node, m.Module, body, seq); err != nil {
m.Published = fmt.Sprintf("not published (tried again at the next event): %v", err)
return m, false
}
m.Published = MembershipPublished
return m, true
}
// WithoutRetired is a membership without the seat verbs it serves that mesh-sdk's seat catalogue no longer has (a verb
// retired on purpose, such as the delivery seat's mark-ready, ADR 0421), and those verbs, `<seat> <verb>`, once each. A
// seat the catalogue does not define (a module's own) keeps every verb, so its loss is still refused. Pure.
func WithoutRetired(last []byte) ([]byte, []string) {
var m map[string]any
if json.Unmarshal(last, &m) != nil {
return last, nil
}
seats, ok := m["seats"].([]any)
if !ok {
return last, nil
}
var kept []any
var retired []string
for _, e := range seats {
served, _ := e.(map[string]any)
seat, _ := served["seat"].(string)
verb, _ := served["verb"].(string)
if retiredVerb(seat, verb) {
if r := seat + " " + verb; !slices.Contains(retired, r) {
retired = append(retired, r)
}
continue
}
kept = append(kept, e)
}
if len(retired) == 0 {
return last, nil
}
if len(kept) == 0 {
delete(m, "seats") // as an empty list is written: not at all
} else {
m["seats"] = kept
}
out, err := json.Marshal(m)
if err != nil {
return last, nil
}
sort.Strings(retired)
return out, retired
}
// retiredVerb says the seat catalogue defines seat with its verbs and no longer has verb among them. A seat the catalogue
// defines without verbs (its holder declares them) retires none.
func retiredVerb(seat, verb string) bool {
if seat == "" || verb == "" {
return false
}
s, ok := catalogue.SeatNamed(seat)
if !ok || len(s.Serves) == 0 {
return false
}
return !slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb })
}
// sameMembership says two memberships say the same, whatever the order of their fields. Pure.
func sameMembership(was, now []byte) bool {
var a, b any
if json.Unmarshal(was, &a) != nil || json.Unmarshal(now, &b) != nil {
return false
}
return reflect.DeepEqual(a, b)
}
// ShadowDifference opens what a membership's pass says when it was not published because it would lose something.
const ShadowDifference = "not published, a shadow difference: publishing would lose "
// Losses names everything the last membership holds that now drops or weakens: a field gone, a value changed, a list
// element (a capability, a kind, a served address, a seat, a state, a stream) with no element in now that holds all of it.
// Empty means now holds all the last one did, and maybe more. Pure.
func Losses(last, now []byte) []string {
var a, b any
if json.Unmarshal(last, &a) != nil {
return nil
}
if json.Unmarshal(now, &b) != nil {
return []string{"the new membership is not JSON"}
}
return losses("", a, b)
}
func losses(path string, last, now any) []string {
switch l := last.(type) {
case map[string]any:
n, ok := now.(map[string]any)
if !ok {
return []string{fmt.Sprintf("%s: %s", name(path), brief(last))}
}
keys := make([]string, 0, len(l))
for k := range l {
keys = append(keys, k)
}
sort.Strings(keys)
var out []string
for _, k := range keys {
nv, ok := n[k]
if !ok {
out = append(out, fmt.Sprintf("%s: %s", name(path+"."+k), brief(l[k])))
continue
}
out = append(out, losses(path+"."+k, l[k], nv)...)
}
return out
case []any:
n, _ := now.([]any)
var out []string
for i, e := range l {
if !slices.ContainsFunc(n, func(c any) bool { return len(losses("", e, c)) == 0 }) {
out = append(out, fmt.Sprintf("%s[%d]: %s", name(path), i, brief(e)))
}
}
return out
}
if !reflect.DeepEqual(last, now) {
return []string{fmt.Sprintf("%s: %s, now %s", name(path), brief(last), brief(now))}
}
return nil
}
func name(path string) string {
if path == "" {
return "the membership"
}
return strings.TrimPrefix(path, ".")
}
// brief is a value in JSON, cut at 200 bytes.
func brief(v any) string {
raw, _ := json.Marshal(v)
if len(raw) > 200 {
return string(raw[:200]) + "…"
}
return string(raw)
}
-414
View File
@@ -1,414 +0,0 @@
package grants
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The part's own states, as its manifest declares them, and the keys of the two that hold one value.
const (
StateUserList = "user-list"
StateMemberships = "memberships"
StateAssertions = "assertions"
StateTakeover = "takeover"
KeyList = "list"
KeySwitch = "switch"
// KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList.
KeyPass = "pass"
KeyAsked = "asked"
)
// States is every state the part keeps.
var States = []string{StateAssertions, StateMemberships, StateTakeover, StateUserList}
// The events the part says.
const (
EventUserList = "user-list-changed"
EventMembership = "membership-changed"
// EventAsserted is said when the part created a bucket, log or stream a manifest declares (issue 1790).
EventAsserted = "asserted"
)
// Emits is every event the part says.
var Emits = []string{EventAsserted, EventMembership, EventUserList}
// Shadow is the switch's one state in this milestone: the bus node follows the controller's list.
const Shadow = "shadow"
// Store is the part's own state, by local name: the only thing the part writes.
type Store interface {
Get(state, key string) ([]byte, uint64, bool, error)
Put(state, key string, value []byte) (uint64, error)
}
// Emitter says one of the part's events.
type Emitter interface {
Emit(event string, body any) error
}
// UserListState is the value of `user-list` in shadow: the list the grants part composed (users and grants, no hashes),
// what the bus loaded, and how they differ. Inputs is the digest of all it says but At: a value is written again only
// when it changes.
type UserListState struct {
V int `json:"v"`
Mode string `json:"mode"`
Inputs string `json:"inputs"`
Digest string `json:"digest"`
Users int `json:"users"`
// LeftOut are the assignments the shadow list leaves out, each with why.
LeftOut []Gap `json:"left_out"`
Compare Comparison `json:"compare"`
Loaded *delivery.UserListWritten `json:"loaded,omitempty"`
// Credentials says per user whether the vault holds its hash and how its line would differ (ADR 0547 clause 9).
Credentials Credentials `json:"credentials"`
BusNode string `json:"bus_node,omitempty"`
Gaps []Gap `json:"gaps"`
At time.Time `json:"at"`
}
func (s UserListState) inputs() string {
s.At, s.Inputs = time.Time{}, ""
raw, _ := json.Marshal(s)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// Result is what one composition did.
type Result struct {
UserList UserListState `json:"user_list"`
Revision uint64 `json:"revision"`
Written bool `json:"written"`
Memberships int `json:"memberships"`
MembershipsWritten int `json:"memberships_written"`
Published []string `json:"published,omitempty"`
NotPublished []string `json:"not_published,omitempty"`
Assertions int `json:"assertions"`
AssertionsWritten int `json:"assertions_written"`
// Asserted is each module whose declared streams this pass created, `<module> at <digest>: created …`, and
// NotAsserted each creation refused, `<module> <stream>: <why>` (issue 1790).
Asserted []string `json:"asserted,omitempty"`
NotAsserted []string `json:"not_asserted,omitempty"`
// Refused is every manifest the indexer holds that this pass could not read, `manifest of <module>: <why>`: its
// module counts as placed nowhere (issue 1945).
Refused []string `json:"refused,omitempty"`
// RootNotFree is each machine the composition needed a root-free word for and did not get one, `<machine>: not
// free, <why>`: a verified sender is composed as from no root-free machine there (issue 1945).
RootNotFree []string `json:"root_not_free,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
}
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
const NothingTakesEffect = "memberships (on the assignments stream, a first one or one that only adds to the last published or drops only a verb the seat catalogue retired; any other loss stays a shadow difference), and each bucket, log or stream a manifest declares that the bus lacks, created once with the manifest's limits (the operator, 2026-10-26 08:54); the user list, the grants and any change to a stream the bus holds stay in shadow until take-over"
// Part is the grants part: it reads, calls and writes through these, and nothing else.
type Part struct {
Read Reader
Call Caller
Store Store
Emit Emitter
// Publish publishes memberships: the process's connection; nil in the tools, which ask the process instead.
Publish Publisher
// Create creates a declared stream the bus lacks: the process's connection; nil in the tools, which ask the process.
Create Creator
Now func() time.Time
cache Manifests
mu sync.Mutex
last Result
made creations
// askSeen is the revision of the last ask to compose (KeyAsked) this part acted on.
askSeen uint64
}
// Ask is the value of KeyAsked: a tool's ask to compose now, and, from grants_assert, to create one module's missing
// streams again even where they were created at this manifest before.
type Ask struct {
By string `json:"by"`
At time.Time `json:"at"`
Assert string `json:"assert,omitempty"`
}
// askedAssert is the module a new ask names to assert by hand, read once per ask; empty when none.
func (p *Part) askedAssert() string {
raw, rev, found, err := p.Store.Get(StateUserList, KeyAsked)
if err != nil || !found || rev == p.askSeen {
return ""
}
p.askSeen = rev
var a Ask
_ = json.Unmarshal(raw, &a)
return a.Assert
}
func (p *Part) now() time.Time {
if p.Now == nil {
return time.Now().UTC()
}
return p.Now().UTC()
}
// Compose composes the user list, the memberships and the assertions from what the bus holds now, compares them, and
// writes each of its own keys whose value changed, saying each change. It never stops at a refusal: a read refused is
// a gap, a write refused is said in Errors, and the next event composes again.
func (p *Part) Compose() Result {
// No lock is held across a round trip on the bus: the reads, calls, publishes and writes below take none, and the
// lock is taken only to keep the result, so a tool answers while a pass waits on the bus.
res := Result{PublishedOrAsserted: NothingTakesEffect}
defer func() {
p.mu.Lock()
p.last = res
p.mu.Unlock()
}()
in := ReadInputs(p.Read, &p.cache)
for _, g := range in.Refused {
res.Refused = append(res.Refused, g.Input+": "+g.Why)
}
recs, left := Records(in)
res.RootNotFree = WithRootFree(&recs, p.Call)
st := UserListState{V: 1, Mode: Shadow, LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}
if st.LeftOut == nil {
st.LeftOut = []Gap{}
}
users, err := ComposeUsers(recs)
if err != nil {
res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
}
users = WithHashes(users, in.Hashes)
st.Digest, st.Users = Digest(users), len(users)
loaded, err := LoadedUsers(p.Call)
st.Compare = Compare(users, loaded, Placed(in))
if err != nil {
st.Compare.Unread = err.Error()
}
st.Credentials = CredentialsOf(users, loaded, in.Hashes)
st.Inputs = st.inputs()
res.UserList = st
if raw, rev, found, err := p.Store.Get(StateUserList, KeyList); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be read: %v", StateUserList, err))
} else {
var was UserListState
if found && json.Unmarshal(raw, &was) == nil && was.Inputs == st.Inputs {
res.Revision = rev
} else if data, err := json.Marshal(st); err == nil {
if rev, err := p.Store.Put(StateUserList, KeyList, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be written: %v", StateUserList, err))
} else {
res.Revision, res.Written = rev, true
p.say(&res, EventUserList, map[string]any{"digest": st.Digest, "revision": rev, "users": st.Users,
"differing": len(st.Compare.Differing()), "mode": Shadow})
}
}
}
ms := Memberships(recs)
res.Memberships = len(ms)
for _, key := range sortedKeys(ms) {
m, now := publish(p.Publish, ms[key])
switch {
case now:
res.Published = append(res.Published, key)
p.say(&res, EventMembership, map[string]any{"node": m.Node, "module": m.Module, "digest": m.Digest, "published": true})
case m.Published != MembershipUnchanged:
res.NotPublished = append(res.NotPublished, key+": "+m.Published)
}
data, _ := json.Marshal(m)
if p.changed(&res, StateMemberships, key, data) {
res.MembershipsWritten++
}
}
held, heldErr := HeldStreams(p.Call)
forced := ""
if p.Create != nil {
forced = p.askedAssert()
}
for _, module := range sortedKeys(in.Manifests) {
if !assigned(in, module) {
continue
}
a := Assert(module, in.Digests[module], in.Manifests[module], held, heldErr)
// Created only where the bus was read and lacks it, never changed where it holds it (issue 1790).
if p.Create != nil && heldErr == nil {
if made := createMissing(p.Create, &p.made, &a, in.Manifests[module], forced == module); len(made) > 0 {
res.Asserted = append(res.Asserted, fmt.Sprintf("%s at %s: created %s", module, a.Manifest, strings.Join(made, ", ")))
p.say(&res, EventAsserted, map[string]any{"module": module, "manifest": a.Manifest, "created": made})
}
for _, why := range notCreated(a) {
res.NotAsserted = append(res.NotAsserted, module+" "+why)
}
}
res.Assertions++
data, _ := json.Marshal(a)
if p.changed(&res, StateAssertions, module, data) {
res.AssertionsWritten++
}
}
if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found {
data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}})
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be written: %v", StateTakeover, err))
}
}
return res
}
// Last is the result of the last pass this part made, kept under the lock alone.
func (p *Part) Last() Result {
p.mu.Lock()
defer p.mu.Unlock()
return p.last
}
func assigned(in Inputs, module string) bool {
for _, ms := range in.Assigned {
for _, m := range ms {
if m == module {
return true
}
}
}
return false
}
// changed writes key when its value differs from the one held, and says whether it wrote.
func (p *Part) changed(res *Result, state, key string, data []byte) bool {
raw, _, found, err := p.Store.Get(state, key)
if err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", state, key, err))
return false
}
if found && bytes.Equal(raw, data) {
return false
}
if _, err := p.Store.Put(state, key, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", state, key, err))
return false
}
return true
}
func (p *Part) say(res *Result, event string, body any) {
if p.Emit == nil {
return
}
if err := p.Emit.Emit(event, body); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("the event %s could not be said: %v", event, err))
}
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// Switch is the value of `takeover`: which list the bus node follows, and every act on the switch.
type Switch struct {
V int `json:"v"`
State string `json:"state"`
Acts []Act `json:"acts"`
}
// Act is one take-over or hand-back asked, done or refused.
type Act struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
Refused string `json:"refused,omitempty"`
}
// keptActs is how many acts the switch keeps, newest last.
const keptActs = 50
// Operator says whether a caller is the operator's credential: a person's own bus account (`person.<name>`), which no
// session of an agent holds (an agent calls as its machine's tool runner).
func Operator(caller string) bool {
name, ok := strings.CutPrefix(caller, "person.")
return ok && name != ""
}
// NotComposable is why take-over is refused in this milestone, whatever the lists say.
const NotComposable = "the enrolment users and the persons cannot be composed: they live only in the controller's store, and take-over waits until they have records on the bus (ADR 0547 clause 9, version 26)"
// record keeps an act on the switch; a refused write is said in the answer.
func (p *Part) record(a Act) string {
raw, _, found, err := p.Store.Get(StateTakeover, KeySwitch)
if err != nil {
return fmt.Sprintf("; the act is not recorded: %v", err)
}
s := Switch{V: 1, State: Shadow}
if found {
_ = json.Unmarshal(raw, &s)
}
s.Acts = append(s.Acts, a)
if len(s.Acts) > keptActs {
s.Acts = s.Acts[len(s.Acts)-keptActs:]
}
data, _ := json.Marshal(s)
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
return fmt.Sprintf("; the act is not recorded: %v", err)
}
return ""
}
// TakeOver is the operator's switch to the grants part's list. It is refused without the operator's credential, while
// any user of the shadow list differs from the loaded list (but those both agree are gone), and in this milestone
// always, since the enrolment users and the persons cannot be composed. Every ask is recorded.
func (p *Part) TakeOver(caller, why string) error {
a := Act{Act: "take-over", By: caller, Why: why, At: p.now()}
if !Operator(caller) {
a.Refused = fmt.Sprintf("take-over is the operator's alone: %s is not a person's account", caller)
return fmt.Errorf("%s%s", a.Refused, p.record(a))
}
res := p.Compose()
var reasons []string
if c := res.UserList.Compare; !c.Equal() {
if c.Unread != "" {
reasons = append(reasons, "the loaded list could not be read: "+c.Unread)
}
if d := c.Differing(); len(d) > 0 {
reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list: %s", len(d), strings.Join(d, ", ")))
}
}
cr, gone := res.UserList.Credentials, res.UserList.Compare.Gone
var unhashed []string
for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) {
if !slices.Contains(gone, u) {
unhashed = append(unhashed, u)
}
}
if len(unhashed) > 0 {
sort.Strings(unhashed)
reasons = append(reasons, fmt.Sprintf("the vault holds no hash for %d users (ADR 0547 clause 9: handed or adopted before take-over): %s",
len(unhashed), strings.Join(unhashed, ", ")))
}
reasons = append(reasons, NotComposable)
a.Refused = "take-over refused: " + strings.Join(reasons, "; ")
return fmt.Errorf("%s%s", a.Refused, p.record(a))
}
// HandBack is the operator's switch back to the controller's list. Refused without the operator's credential, and
// while the part is in shadow, since the bus node follows the controller's list already. Every ask is recorded.
func (p *Part) HandBack(caller, why string) error {
a := Act{Act: "hand-back", By: caller, Why: why, At: p.now()}
if !Operator(caller) {
a.Refused = fmt.Sprintf("hand-back is the operator's alone: %s is not a person's account", caller)
return fmt.Errorf("%s%s", a.Refused, p.record(a))
}
a.Refused = "nothing to hand back: the grants part is in shadow and never took over, so the bus node follows the controller's list already"
return fmt.Errorf("%s%s", a.Refused, p.record(a))
}
@@ -1,273 +0,0 @@
package grants
import (
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Records is what the bus composer composes from, rebuilt from the bus (mesh-controller internal/inventory busrecords.go,
// BusRecords, read the same way from Inputs instead of the controller's store). Pure. An assigned module whose manifest
// the indexer does not hold is left out and named, where the controller refuses the whole composition: the shadow
// composes what it can and says what it could not.
func Records(in Inputs) (buscompose.Records, []Gap) {
var gaps []Gap
seats := map[string]catalogue.SeatDeclaration{}
declarers := map[string]string{}
for _, m := range in.Manifests {
for _, s := range m.DefinesSeats {
seats[s.Name] = s
declarers[s.Name] = m.Module
}
}
for _, own := range catalogue.SeatsWithAProtocol() {
seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves}
}
out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{},
Interchangeable: map[string]bool{}}
values := func(provider, setting string) []any {
var vs []any
for _, node := range in.Nodes {
if v, ok := in.Values[node+"/"+provider+"/"+setting]; ok && slices.Contains(in.Assigned[node], provider) {
vs = append(vs, v)
}
}
return vs
}
for _, node := range in.Nodes {
out.Nodes = append(out.Nodes, node)
manifestOf := func(module string) (catalogue.Manifest, bool) {
if m, ok := in.Running[node+"."+module]; ok {
return m, true
}
m, ok := in.Manifests[module]
return m, ok
}
var here []catalogue.Manifest
for _, module := range in.Assigned[node] {
if m, ok := manifestOf(module); ok {
here = append(here, m)
}
}
for _, module := range in.Assigned[node] {
m, known := manifestOf(module)
if !known {
gaps = append(gaps, Gap{Input: module + " on " + node, Why: "left out of the shadow list: no manifest to derive what it may say from"})
continue
}
d := declaredFor(m, seats, declarers)
for _, sr := range catalogue.ReadsGranted(m, here, node) {
d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key})
}
d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes)
out.Assigned[node] = append(out.Assigned[node], d)
if m.Instances == catalogue.InstancesInterchangeable {
out.Interchangeable[m.Module] = true
}
}
}
return out, gaps
}
// declaredFor is mesh-controller's declaredFor (internal/inventory/busrecords.go), copied: one module's manifest as the
// composer needs it.
func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration, declarers map[string]string) buscompose.Declared {
var fromModules []string
var watches []buscompose.Seat
for _, c := range m.Consumes {
emitter, event, named := strings.Cut(c, ".")
if named {
if s, isASeat := seats[emitter]; isASeat && s.Kinded {
ev := strings.TrimSuffix(event, ".*")
if catalogue.SeatSays(s.Emits, ev) {
watches = append(watches, buscompose.Seat{Name: s.Name, Scope: s.Scope, Emits: []string{ev},
Kinded: true, Proofs: s.Proofs, DeclaredBy: declarers[s.Name]})
continue
}
}
if s, isASeat := seats[emitter]; isASeat && catalogue.SeatSays(s.Emits, event) {
watches = append(watches, buscompose.Seat{Name: s.Name, Emits: []string{event}})
continue
}
}
fromModules = append(fromModules, c)
}
d := buscompose.Declared{
Module: m.Module, Emits: m.EmitsAll(), Consumes: fromModules, Watches: watches, Serves: m.Tools,
Invokes: m.Invokes, Waits: waitsOf(m, seats), State: bucketsOf(m), Reads: m.Reads, Logs: logsOf(m),
Streams: streamsOf(m), Queues: queueOwnersOf(m), Processes: processesOf(m), Checks: catalogue.HealthChecks(m),
RunsAs: m.RunsAs, Issued: issuedTools(m),
}
for _, b := range m.Bundles {
if len(b.Loads) > 0 {
d.Loads = true
}
}
if _, reads := m.OwnSecrets["broker"]; !reads {
d.NoAccount = true
}
if _, reads := m.OwnSecrets[buscompose.ProvisionerSecret]; reads && m.ClaimsSeat(buscompose.NodeProvisionerSeat) {
d.ProvisionsNode = true
}
d.SnapshotsTheBus = m.ClaimsSeat(catalogue.BrokerSeat)
for _, c := range m.Claims {
if s, hasAProtocol := seats[c.Name]; hasAProtocol {
held := asSeat(s, declarers[s.Name])
held.Kind = c.Kind
held.Capabilities = c.Capabilities
d.Holds = append(d.Holds, held)
}
}
for _, name := range m.Uses {
if s, declaredSomewhere := seats[name]; declaredSomewhere {
d.Uses = append(d.Uses, asSeat(s, declarers[s.Name]))
}
}
return d
}
func waitsOf(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) []buscompose.Wait {
var out []buscompose.Wait
for _, w := range m.Waits {
emitter, event, _ := strings.Cut(w, ".")
first, _, _ := strings.Cut(event, ".")
wait := buscompose.Wait{Name: w, Emitter: emitter, Event: event}
if s, isASeat := seats[emitter]; isASeat {
switch {
case s.Kinded && catalogue.SeatSays(s.Emits, first):
wait.Seat, wait.Last = true, "kind"
case s.NamedByCaller(first) && catalogue.SeatSays(s.Emits, first):
wait.Seat, wait.Last = true, "asker"
case catalogue.SeatSays(s.Emits, event):
wait.Seat = true
}
}
out = append(out, wait)
}
return out
}
func issuedTools(m catalogue.Manifest) []string {
var out []string
for _, o := range m.Provides {
if tool, issued := m.IssuedCredentialOf(o.Name); issued && !slices.Contains(out, tool) {
out = append(out, tool)
}
}
return out
}
func bucketsOf(m catalogue.Manifest) []buscompose.Bucket {
var out []buscompose.Bucket
for _, s := range m.State {
out = append(out, buscompose.Bucket{Module: m.Module, Name: s.Name, History: s.History, TTLSeconds: s.TTLSeconds,
PerMachine: s.PerMachine, KeyTTL: s.KeyTTL})
}
return out
}
func logsOf(m catalogue.Manifest) []buscompose.Log {
var out []buscompose.Log
for _, l := range m.Logs {
out = append(out, buscompose.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap()})
}
return out
}
func asSeat(s catalogue.SeatDeclaration, declarer string) buscompose.Seat {
seat := buscompose.Seat{Name: s.Name, Scope: s.Scope, Accepts: s.Accepts, Emits: s.Emits,
Serves: catalogue.VerbNames(s.Serves), Kinded: s.Kinded, ByCaller: s.ByCaller, Proofs: s.Proofs, DeclaredBy: declarer}
if own, ok := catalogue.SeatNamed(s.Name); ok && own.PulledByKind {
seat.PulledByKind = true
}
for _, r := range s.Records {
if declarer != "" {
seat.Records = append(seat.Records, buscompose.BucketName(declarer, r))
}
}
return seat
}
func streamsOf(m catalogue.Manifest) []buscompose.ModuleStream {
var out []buscompose.ModuleStream
for _, s := range m.Streams {
events := make([]string, 0, len(s.Events))
for _, e := range s.Events {
events = append(events, buscompose.WirePattern(e))
}
out = append(out, buscompose.ModuleStream{Module: m.Module, Name: s.BusName(), Events: events,
MaxAgeDays: s.Days(), MaxMiB: s.Cap(), Restricted: s.Grant != nil,
WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue, Duplicates: s.Duplicates})
}
return out
}
func queueOwnersOf(m catalogue.Manifest) []buscompose.QueueOwner {
buses := m.ProcessBuses()
if len(buses) == 0 {
return nil
}
byName := map[string]catalogue.StreamDeclaration{}
for _, s := range m.Streams {
byName[s.Name] = s
}
var out []buscompose.QueueOwner
for _, r := range m.Resources {
id := fmt.Sprint(r["id"])
b, declared := buses[id]
if !declared {
continue
}
for _, q := range b.Queues {
s, known := byName[q]
if !known || s.Retention != catalogue.StreamRetentionWorkQueue {
continue
}
out = append(out, buscompose.QueueOwner{Process: id, Stream: s.BusName(), MaxDeliver: s.MaxDeliveries})
}
}
return out
}
func processesOf(m catalogue.Manifest) []buscompose.ProcessPrincipal {
buses := m.ProcessBuses()
if len(buses) == 0 {
return nil
}
queues := queueOwnersOf(m)
var out []buscompose.ProcessPrincipal
for _, r := range m.Resources {
id := fmt.Sprint(r["id"])
b, declared := buses[id]
if !declared {
continue
}
pr := buscompose.ProcessPrincipal{Process: id, Emits: b.Emits, State: b.State, Asks: b.Asks, Invokes: b.Invokes}
for _, q := range queues {
if q.Process == id {
pr.Queues = append(pr.Queues, q)
}
}
out = append(out, pr)
}
return out
}
func hearsOf(m catalogue.Manifest, declared map[string]catalogue.Manifest, node string, values func(provider, setting string) []any,
consumes []string) ([]buscompose.Heard, bool, []string) {
reading := m
reading.Consumes = consumes
parts, touches, onEvents := catalogue.Hearing(reading, declared, node, values)
var out []buscompose.Heard
for _, p := range parts {
h := buscompose.Heard{Stream: p.Stream.BusName(), Emitter: p.Emitter, Restricted: p.Stream.Grant != nil}
for _, pattern := range p.Patterns {
h.Filters = append(h.Filters, buscompose.ModuleStreamSubject(p.Emitter, buscompose.WirePattern(pattern)))
}
out = append(out, h)
}
return out, touches, onEvents
}
@@ -1,122 +0,0 @@
package grants
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// CallRootFree is the controller seat's verb that judges machines root-free (novox/hq ADR 0259 §8): the one judgement
// the composer needs and the part cannot make from the bus (issue 1945). Only the controller answers it, on purpose.
const CallRootFree = "seat:mesh-controller.root-free"
// The controller seat's condition events the part hears (never a timer): a machine's root-free word changes with the
// raising and the clearing of its `root-not-free` condition.
const (
EventConditionRaised = "mesh-controller.condition-raised"
EventConditionCleared = "mesh-controller.condition-cleared"
// KindRootNotFree is the condition kind whose events wake a pass.
KindRootNotFree = "root-not-free"
)
// Consumes is every event the part hears, as its manifest's consumes name them.
var Consumes = []string{EventConditionRaised, EventConditionCleared}
// EventSubject is the bus subject of a controller seat event named `mesh-controller.<event>`.
func EventSubject(event string) string {
return "mesh.seat.mesh-controller.event." + strings.TrimPrefix(event, "mesh-controller.")
}
// IsRootNotFree says a condition event's body names the `root-not-free` kind (its `kind`, or the last part of its
// `key`, `<scope>.<id>.<kind>`). An unreadable body wakes a pass too: a pass is cheap, a missed change is not.
func IsRootNotFree(body []byte) bool {
var c struct {
Kind string `json:"kind"`
Key string `json:"key"`
}
if json.Unmarshal(body, &c) != nil {
return true
}
return c.Kind == KindRootNotFree || (c.Kind == "" && strings.HasSuffix(c.Key, "."+KindRootNotFree)) || (c.Kind == "" && c.Key == "")
}
// RootFreeNodes are the machines a composition needs a root-free word for: where the router's seat is held and where a
// kinded seat's holder declares `verified-sender`. None when no holder declares it. Sorted. Pure.
func RootFreeNodes(r buscompose.Records) []string {
var nodes []string
need := false
for node, declared := range r.Assigned {
for _, d := range declared {
for _, s := range d.Holds {
if s.Name == "operator-channel" && !slices.Contains(nodes, node) {
nodes = append(nodes, node)
}
if s.Kinded && s.Kind != "" && slices.Contains(s.Capabilities, "verified-sender") {
need = true
if !slices.Contains(nodes, node) {
nodes = append(nodes, node)
}
}
}
}
}
if !need {
return nil
}
sort.Strings(nodes)
return nodes
}
// verdict is one machine's answer of the controller's root-free verb.
type verdict struct {
Machine string `json:"machine"`
Free bool `json:"free"`
Why string `json:"why"`
}
// WithRootFree fills r.RootFree for the machines the composition needs by asking the controller once, and answers a
// line for each machine that is not free, `<machine>: <why>`. It fails closed: an error, no answer or a machine
// left out is not free, and says so. Pure but for the one call; the answer is kept for this pass alone.
func WithRootFree(r *buscompose.Records, call Caller) []string {
nodes := RootFreeNodes(*r)
if len(nodes) == 0 {
return nil
}
free := map[string]verdict{}
var failure string
if call == nil {
failure = "no way to ask the controller here"
} else if raw, err := call.Call(CallRootFree, map[string]any{"machines": nodes}); err != nil {
failure = "the controller's root-free word could not be asked: " + err.Error()
} else {
var body struct {
Machines []verdict `json:"machines"`
}
if err := json.Unmarshal(raw, &body); err != nil {
failure = "the controller's root-free answer could not be read: " + err.Error()
}
for _, v := range body.Machines {
free[v.Machine] = v
}
}
r.RootFree = map[string]bool{}
var notes []string
for _, n := range nodes {
v, answered := free[n]
switch {
case answered && v.Free:
r.RootFree[n] = true
case failure != "":
notes = append(notes, fmt.Sprintf("%s: not free, %s", n, failure))
case !answered:
notes = append(notes, fmt.Sprintf("%s: not free, the controller named no verdict for it", n))
default:
notes = append(notes, fmt.Sprintf("%s: not free, %s", n, v.Why))
}
}
return notes
}
@@ -1,243 +0,0 @@
package grants
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The part's tools, as its manifest lists them. Module tools in this milestone: a `mesh-*` seat must be in mesh-sdk's own
// seat list, which the running controller's composer does not know, so the part claims no seat until take-over.
const (
ToolGrants = "grants_grants"
ToolMembership = "grants_membership"
ToolUserList = "grants_user_list"
ToolComposeNow = "grants_compose_now"
ToolAssert = "grants_assert"
ToolTakeOver = "grants_take_over"
ToolHandBack = "grants_hand_back"
)
// Tools is every tool, in the manifest's order.
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack}
// WritesNoGrant is what grants says of the part's own grants in this milestone.
const WritesNoGrant = "none: the grants part writes no grants.<module>.<fp> before ADR 0547's third milestone (clause 5); the controller's are beside"
// Grants answers the tool grants: what the part writes of grants (none), and the controller's, one module's or all.
func (p *Part) Grants(module string) (any, error) {
keys, err := p.Read.Keys(ReadControllerGrant)
if err != nil {
return map[string]any{"writes": WritesNoGrant, "controller_unread": err.Error()}, nil
}
type grant struct {
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
Accounts []string `json:"accounts,omitempty"`
}
out := []grant{}
for _, k := range keys {
m, fp, err := delivery.ParseControllerGrantKey(k)
if err != nil || (module != "" && m != module) {
continue
}
g := grant{Module: m, Fingerprint: fp}
if module != "" {
if raw, found, err := p.Read.Get(ReadControllerGrant, k); err == nil && found {
if v, err := delivery.DecodeControllerGrant(raw); err == nil {
g.Accounts = v.Accounts
}
}
}
out = append(out, g)
}
sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Fingerprint < out[j].Module+out[j].Fingerprint })
return map[string]any{"writes": WritesNoGrant, "controller": out, "count": len(out)}, nil
}
// MembershipOf answers the tool membership: one module on one node as composed now, and as last published by the
// part's process (its state memberships says what became of each at the last pass).
func (p *Part) MembershipOf(node, module string) (any, error) {
if node == "" && module == "" {
return p.Held()
}
if node == "" || module == "" {
return nil, errors.New("membership names a node and a module, or neither for every membership held back")
}
in := ReadInputs(p.Read, &p.cache)
recs, left := Records(in)
notFree := WithRootFree(&recs, p.Call)
m, ok := Memberships(recs)[node+"."+module]
out := map[string]any{"node": node, "module": module, "gaps": MembershipGaps, "root_free": recs.RootFree, "root_not_free": notFree}
if ok {
out["composed"] = m
} else {
why := fmt.Sprintf("%s is not assigned to %s on the bus", module, node)
for _, g := range left {
if g.Input == module+" on "+node {
why = g.Why
}
}
out["composed"] = why
}
if raw, rev, found, err := p.Store.Get(StateMemberships, node+"."+module); err != nil {
out["published"] = "not readable: " + err.Error()
} else if found {
out["published"] = map[string]any{"revision": rev, "last_pass": json.RawMessage(raw)}
} else {
out["published"] = "no pass of the part's process has composed it yet"
}
return out, nil
}
// Held answers every membership the part's process held back at its last pass, each `<node>.<module>` with why (what
// publishing would lose), from the pass it keeps (issue 1945).
func (p *Part) Held() (any, error) {
raw, rev, found, err := p.Store.Get(StateUserList, KeyPass)
if err != nil {
return nil, fmt.Errorf("the last pass cannot be read: %w", err)
}
if !found {
return map[string]any{"held": []any{}, "says": "no pass of the part's process has been kept yet"}, nil
}
var res Result
if err := json.Unmarshal(raw, &res); err != nil {
return nil, fmt.Errorf("the last pass cannot be read: %w", err)
}
type held struct {
Membership string `json:"membership"`
Why string `json:"why"`
}
out := []held{}
for _, n := range res.NotPublished {
key, why, _ := strings.Cut(n, ": ")
out = append(out, held{Membership: key, Why: why})
}
refused := res.Refused
if refused == nil {
refused = []string{}
}
return map[string]any{"held": out, "count": len(out), "pass": rev, "published": res.Published, "refused": refused}, nil
}
// Pass composes and keeps the result as the last pass, for the tools to read.
func (p *Part) Pass() Result {
res := p.Compose()
data, _ := json.Marshal(res)
if _, err := p.Store.Put(StateUserList, KeyPass, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyPass, err))
}
return res
}
// NoGrantAgain is what compose-now says of the controller's grants: the controller serves no verb that writes a module's
// grant again (its issue replaces the account, which is not that), so a grant write it lost stays lost until it composes
// again; the part writes grants only from ADR 0547's third milestone.
const NoGrantAgain = "missing: a verb to ask the controller to write a module's grant again (mesh-controller serves none; its issue replaces the bus account, which is not that), so compose-now cannot ask it"
// AskCompose is the tool compose-now: it asks the part's process to compose and publish now (only the process holds the
// grant to publish a membership), and answers the last pass, whose revision moves when the process answered.
func (p *Part) AskCompose(caller string) (any, error) {
ask, _ := json.Marshal(Ask{By: caller, At: p.now()})
rev, err := p.Store.Put(StateUserList, KeyAsked, ask)
if err != nil {
return nil, fmt.Errorf("the ask could not be written: %w", err)
}
out := map[string]any{"asked": rev, "says": "the part's process composes and publishes at this ask; grants_user_list's last pass says what it published",
"controller_grants": NoGrantAgain}
if raw, prev, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
out["last_pass"] = map[string]any{"revision": prev, "result": json.RawMessage(raw)}
}
return out, nil
}
// UserList answers the tool user-list: the shadow list as last written, beside what the bus node says it loaded now.
func (p *Part) UserList() (any, error) {
out := map[string]any{"mode": Shadow}
raw, rev, found, err := p.Store.Get(StateUserList, KeyList)
switch {
case err != nil:
out["written"] = "not readable: " + err.Error()
case !found:
out["written"] = "nothing composed yet: grants_compose_now composes it"
default:
var st UserListState
if err := json.Unmarshal(raw, &st); err != nil {
return nil, fmt.Errorf("its own user list cannot be read: %w", err)
}
out["revision"], out["written"] = rev, st
if raw, prev, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
out["last_pass"] = map[string]any{"revision": prev, "result": json.RawMessage(raw)}
}
if st.BusNode != "" {
if raw, found, err := p.Read.Get(ReadHave, st.BusNode+"."+BusModule); err == nil && found {
if h, err := delivery.DecodeHave(raw); err == nil {
out["loaded_now"] = h.UserList
}
}
}
}
return out, nil
}
// AssertOne answers the tool assert: one module's buckets, logs and streams as composed from its manifest at the
// indexer's trunk, beside what the bus holds, and asks the part's process (which alone holds the grant to create) to
// create what the bus lacks now, even one it created at this manifest and that is gone since (issue 1790). It changes no
// stream the bus holds. Here, in the tools, it creates nothing itself: the answer names the ask.
func (p *Part) AssertOne(module string) (any, error) {
if module == "" {
return nil, errors.New("assert names a module")
}
raw, found, err := p.Read.Get(ReadModules, module)
if err != nil {
return nil, fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if !found {
return nil, fmt.Errorf("the indexer holds no module %s", module)
}
v, err := delivery.DecodeModule(raw)
if err != nil || v.Manifest == "" {
return nil, fmt.Errorf("the indexer names no manifest of %s", module)
}
m, err := p.cache.get(p.Read, v.Manifest)
if err != nil {
return nil, err
}
held, heldErr := HeldStreams(p.Call)
a := Assert(module, v.Manifest, m, held, heldErr)
out := map[string]any{"assertion": a}
missing := 0
for _, it := range a.Items {
if it.Held == nil {
missing++
}
}
if missing == 0 {
out["says"] = "the bus holds every bucket, log and stream the manifest declares: nothing to create; a difference is only said"
return out, nil
}
ask, _ := json.Marshal(Ask{By: "grants_assert", At: p.now(), Assert: module})
rev, err := p.Store.Put(StateUserList, KeyAsked, ask)
if err != nil {
return nil, fmt.Errorf("the ask to create could not be written: %w", err)
}
out["asked"] = rev
out["says"] = fmt.Sprintf("%d the bus lacks: the part's process creates them at this ask if %s is placed (its grant names only placed "+
"modules' streams) and says asserted; grants_assert again shows what the bus holds then", missing, module)
return out, nil
}
// Answer is a tool's arguments read as text.
func Answer(args map[string]any, name string) string {
switch v := args[name].(type) {
case string:
return strings.TrimSpace(v)
case bool:
return fmt.Sprint(v)
}
return ""
}
@@ -1,327 +0,0 @@
package grants
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// User is one bus user and its grants, as the bus module's nats_users answers them: never a password hash.
type User struct {
User string `json:"user"`
Publish []string `json:"publish"`
Subscribe []string `json:"subscribe"`
AllowResponses bool `json:"allow_responses"`
// Hash is the bcrypt hash the vault holds for the user (WithHashes): in memory alone, never in a digest, a compare
// or the part's state, and empty for a loaded user.
Hash string `json:"-"`
}
// WithHashes gives each user of the shadow list the hash the vault holds for it, when it holds one.
func WithHashes(users []User, hashes map[string]HeldHash) []User {
for i := range users {
users[i].Hash = hashes[users[i].User].Hash
}
return users
}
// Credential is one user's hash as the list would carry it: whether the vault holds one, and how the user stands in the
// shadow list and the loaded list. Never the hash: a bcrypt hash of the same password differs by its salt, so two hashes
// say nothing compared.
type Credential struct {
User string `json:"user"`
InShadow bool `json:"in_shadow"`
Loaded bool `json:"loaded"`
HashHeld bool `json:"hash_held"`
Adopted bool `json:"adopted,omitempty"`
// Generation is the vault's generation of the account the hash is of; zero for an adopted one.
Generation int `json:"generation,omitempty"`
// Would is how the list with the vault's hashes would differ from the loaded one for this user: same (a hash held,
// loaded), adds (a hash held, not loaded), left-out (in the shadow list, no hash: left out for want of a
// credential), loses-login (loaded, no hash held: its login would be lost) or not-composed (a hash held, loaded,
// and not in the shadow list).
Would string `json:"would"`
}
// The ways a user's line would differ.
const (
WouldSame = "same"
WouldAdd = "adds"
WouldLeaveOut = "left-out"
WouldLoseLogin = "loses-login"
WouldNotComposed = "not-composed"
)
// Credentials is every user's Credential, and the users of each way that differs.
type Credentials struct {
Users []Credential `json:"users"`
HashesHeld int `json:"hashes_held"`
LeftOut []string `json:"left_out"`
LosesLogin []string `json:"loses_login"`
Adds []string `json:"adds"`
}
// CredentialsOf says, of every user of the shadow and the loaded list, whether the vault holds its hash and how its line
// would differ. Pure.
func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credentials {
c := Credentials{Users: []Credential{}, LeftOut: []string{}, LosesLogin: []string{}, Adds: []string{}}
rows := map[string]*Credential{}
row := func(user string) *Credential {
if rows[user] == nil {
h, held := hashes[user]
rows[user] = &Credential{User: user, HashHeld: held, Adopted: h.Adopted, Generation: h.Generation}
}
return rows[user]
}
for _, u := range shadow {
row(u.User).InShadow = true
}
for _, u := range loaded {
row(u.User).Loaded = true
}
for _, user := range sortedKeys(rows) {
r := rows[user]
switch {
case r.HashHeld && r.InShadow && r.Loaded:
r.Would = WouldSame
case r.HashHeld && r.InShadow:
r.Would = WouldAdd
c.Adds = append(c.Adds, user)
case r.InShadow:
r.Would = WouldLeaveOut
c.LeftOut = append(c.LeftOut, user)
case !r.HashHeld:
r.Would = WouldLoseLogin
c.LosesLogin = append(c.LosesLogin, user)
default:
r.Would = WouldNotComposed
}
if r.HashHeld {
c.HashesHeld++
}
c.Users = append(c.Users, *r)
}
return c
}
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure.
func ComposeUsers(r buscompose.Records) ([]User, error) {
principals, err := buscompose.Users(r)
if err != nil {
return nil, err
}
out := make([]User, 0, len(principals))
for _, p := range principals {
perms, err := buscompose.PermissionsFor(p)
if err != nil {
return nil, fmt.Errorf("%s: %w", p.Username(), err)
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses})
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, nil
}
func set(in []string) []string {
out := slices.Clone(in)
sort.Strings(out)
out = slices.Compact(out)
if out == nil {
out = []string{}
}
return out
}
// Digest is a list's digest, `sha256:<hex>`, over its users and grants in order.
func Digest(users []User) string {
raw, _ := json.Marshal(users)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// LoadedUsers is the list the bus loaded, as its module's nats_users reads it from the file: every user by name, then
// each one's grants in full.
func LoadedUsers(c Caller) ([]User, error) {
raw, err := c.Call(CallUsers, map[string]any{})
if err != nil {
return nil, fmt.Errorf("%s: %w", CallUsers, err)
}
var names struct {
Users []struct {
User string `json:"user"`
} `json:"users"`
}
if err := json.Unmarshal(unwrap(raw), &names); err != nil {
return nil, fmt.Errorf("%s answered no list of users: %w", CallUsers, err)
}
full := map[string]User{}
ask := func(part string) error {
raw, err := c.Call(CallUsers, map[string]any{"user": part})
if err != nil {
return fmt.Errorf("%s %s: %w", CallUsers, part, err)
}
var got struct {
Users []User `json:"users"`
}
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
return fmt.Errorf("%s %s answered no grants: %w", CallUsers, part, err)
}
for _, u := range got.Users {
u.Publish, u.Subscribe = set(u.Publish), set(u.Subscribe)
full[u.User] = u
}
return nil
}
// Every name with a dot in one call; each name without one (the controller) alone.
if err := ask("."); err != nil {
return nil, err
}
out := make([]User, 0, len(names.Users))
for _, n := range names.Users {
if _, ok := full[n.User]; !ok {
if err := ask(n.User); err != nil {
return nil, err
}
}
u, ok := full[n.User]
if !ok {
return nil, fmt.Errorf("%s names %s and gives no grants for it", CallUsers, n.User)
}
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, nil
}
// unwrap takes a tool's answer out of an MCP content wrapper, when a runtime hands one.
func unwrap(raw json.RawMessage) []byte {
var wrapped struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) == 1 && wrapped.Content[0].Text != "" {
return []byte(wrapped.Content[0].Text)
}
return raw
}
// UserDiff is how one user's grants differ: what the shadow list grants that the loaded list does not, and back.
type UserDiff struct {
User string `json:"user"`
PublishAdded []string `json:"publish_added,omitempty"`
PublishDropped []string `json:"publish_dropped,omitempty"`
SubscribeAdded []string `json:"subscribe_added,omitempty"`
SubscribeDropped []string `json:"subscribe_dropped,omitempty"`
Responses string `json:"responses,omitempty"`
}
// Comparison is the shadow list against the loaded list.
type Comparison struct {
Shadow int `json:"shadow"`
Loaded int `json:"loaded"`
// Added are users in the shadow list the loaded list lacks (one left out for want of a credential among them);
// Dropped are loaded users the shadow list lacks; Gone are those of Dropped whose machine or module placement no
// longer names, which both lists agree are gone.
Added []string `json:"added"`
Dropped []string `json:"dropped"`
Gone []string `json:"gone,omitempty"`
Differ []UserDiff `json:"differ"`
Same int `json:"same"`
// Unread says the loaded list could not be read, and why: nothing is compared then.
Unread string `json:"unread,omitempty"`
}
// Equal says the two lists agree in every user, but those both agree are gone.
func (c Comparison) Equal() bool {
return c.Unread == "" && len(c.Added) == 0 && len(c.Differ) == 0 && len(c.Dropped) == len(c.Gone)
}
// Differing names every user that keeps the lists apart: added, dropped and not gone, or granted otherwise.
func (c Comparison) Differing() []string {
var out []string
out = append(out, c.Added...)
for _, d := range c.Dropped {
if !slices.Contains(c.Gone, d) {
out = append(out, d)
}
}
for _, d := range c.Differ {
out = append(out, d.User)
}
sort.Strings(out)
return out
}
// Compare compares the shadow list with the loaded one. placed says whether placement still names a loaded user's
// machine and module; nil names every one. Pure.
func Compare(shadow, loaded []User, placed func(user string) bool) Comparison {
c := Comparison{Shadow: len(shadow), Loaded: len(loaded), Added: []string{}, Dropped: []string{}, Differ: []UserDiff{}}
have := map[string]User{}
for _, u := range loaded {
have[u.User] = u
}
want := map[string]bool{}
for _, s := range shadow {
want[s.User] = true
l, ok := have[s.User]
if !ok {
c.Added = append(c.Added, s.User)
continue
}
d := UserDiff{User: s.User, PublishAdded: minus(s.Publish, l.Publish), PublishDropped: minus(l.Publish, s.Publish),
SubscribeAdded: minus(s.Subscribe, l.Subscribe), SubscribeDropped: minus(l.Subscribe, s.Subscribe)}
if s.AllowResponses != l.AllowResponses {
d.Responses = fmt.Sprintf("shadow %v, loaded %v", s.AllowResponses, l.AllowResponses)
}
if len(d.PublishAdded)+len(d.PublishDropped)+len(d.SubscribeAdded)+len(d.SubscribeDropped) == 0 && d.Responses == "" {
c.Same++
continue
}
c.Differ = append(c.Differ, d)
}
for _, l := range loaded {
if !want[l.User] {
c.Dropped = append(c.Dropped, l.User)
if placed != nil && !placed(l.User) {
c.Gone = append(c.Gone, l.User)
}
}
}
return c
}
func minus(a, b []string) []string {
var out []string
for _, x := range a {
if !slices.Contains(b, x) {
out = append(out, x)
}
}
return out
}
// Placed says whether placement still names a user's machine and module: a machine's users (`node.<n>`,
// `provisioner.<n>`) while the machine is seen, a module's or process's (`<n>.<module>[.<process>]`) while the module is
// assigned there. The controller, enrolment users and persons are never judged gone from here.
func Placed(in Inputs) func(user string) bool {
return func(user string) bool {
parts := strings.Split(user, ".")
switch {
case len(parts) < 2:
return true
case parts[0] == "node" || parts[0] == "provisioner":
return slices.Contains(in.Nodes, parts[1])
case parts[0] == "person" || parts[0] == "enrol":
return true
}
return slices.Contains(in.Nodes, parts[0]) && slices.Contains(in.Assigned[parts[0]], parts[1])
}
}
-110
View File
@@ -1,110 +0,0 @@
{
"module": "mesh-grants",
"version": "1",
"slug": "grants",
"reads": [
"mesh-config.resolved",
"mesh-controller.assignments",
"mesh-controller.grants",
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"node-engine.have",
"mesh-vault.bus-accounts"
],
"consumes": [
"mesh-controller.condition-raised",
"mesh-controller.condition-cleared"
],
"invokes": [
"nats.nats_streams",
"nats.nats_user_can",
"nats.nats_users",
"seat:mesh-controller.root-free"
],
"emits": [
"asserted",
"membership-changed",
"user-list-changed"
],
"tools": [
"grants_grants",
"grants_membership",
"grants_user_list",
"grants_compose_now",
"grants_assert",
"grants_take_over",
"grants_hand_back"
],
"state": [
{
"name": "assertions",
"history": 5
},
{
"name": "memberships",
"history": 5
},
{
"name": "takeover",
"history": 20
},
{
"name": "user-list",
"history": 20
}
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-grants",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-grants",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-grants"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-grants",
"place": ".",
"unknown": "program"
},
{
"id": "grants",
"type": "process",
"name": "mesh-grants",
"artifact": "program",
"run": [
"./mesh-grants"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker"
},
"user": "mesh-grants",
"health": {
"kind": "unit"
}
}
],
"build": {
"artifacts": [
{
"name": "program",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-grants",
"binary": "mesh-grants",
"loads": [
"mesh-grants"
]
}
]
}
}