Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2caaa6fc92 | ||
|
|
f6df07605d | ||
|
|
7692f201df | ||
|
|
09da8d741e | ||
|
|
b2351a213d | ||
|
|
2abceb1913 | ||
|
|
685810720e | ||
|
|
8e25926e91 | ||
|
|
68628bfd78 | ||
|
|
1dbb7c5740 | ||
|
|
9e3d1d5e68 | ||
|
|
9a70d697af | ||
|
|
a4116a1de7 | ||
|
|
6f721dd536 | ||
|
|
354b9ea396 | ||
|
|
3618831992 | ||
|
|
843a1589d7 | ||
|
|
fc6d5dcbdb | ||
|
|
6d709e1fc1 | ||
|
|
eb11b2c844 | ||
|
|
67e3ebf22f | ||
|
|
9b31746cdb | ||
|
|
ebad58f7e2 | ||
|
|
3c6fb7fc9e | ||
|
|
71aa3e86c6 |
@@ -0,0 +1,7 @@
|
||||
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
|
||||
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
|
||||
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
|
||||
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
|
||||
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
|
||||
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
|
||||
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
|
||||
@@ -177,8 +177,8 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
|
||||
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
|
||||
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
for _, missing := range readBuckets {
|
||||
if missing == delivery.PlacementBucket {
|
||||
continue // it starts without it, below
|
||||
if missing == delivery.PlacementBucket || sourceOfGrants(missing) {
|
||||
continue // it starts without it, below (the grants' source: main_test.go)
|
||||
}
|
||||
b := startBed(t, missing)
|
||||
select {
|
||||
|
||||
@@ -8,6 +8,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -54,6 +55,8 @@ var readBuckets = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
buscompose.LeaseBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
@@ -83,7 +86,8 @@ var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposals
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9).
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
|
||||
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
|
||||
var keptStates = []announce.State{
|
||||
{Name: delivery.PlacementState, Readable: true},
|
||||
{Name: delivery.PlacementProposalsState, Readable: true},
|
||||
@@ -105,8 +109,33 @@ type buckets map[string]jetstream.KeyValue
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
out := buckets{}
|
||||
// The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names
|
||||
// where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the
|
||||
// controller holds it, as before.
|
||||
away, why := false, ""
|
||||
if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err)
|
||||
} else {
|
||||
out[buscompose.LeaseBucket] = kv
|
||||
e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey)
|
||||
var raw []byte
|
||||
if gerr == nil {
|
||||
raw = e.Value()
|
||||
}
|
||||
if errors.Is(gerr, jetstream.ErrKeyNotFound) {
|
||||
gerr = nil
|
||||
}
|
||||
away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr)
|
||||
}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
if name == buscompose.LeaseBucket {
|
||||
continue
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == provider.AssignmentsBucket && err != nil && away {
|
||||
logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why)
|
||||
continue
|
||||
}
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
@@ -117,6 +146,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if sourceOfGrants(name) && err != nil {
|
||||
// The grants part's grants and the controller's lease choose where grants are read from (issue 774): without
|
||||
// them the controller's grants are read, as before the lease is handed away.
|
||||
logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
@@ -125,6 +160,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sourceOfGrants says the bucket is one of the two that choose the grants' source, the grants part's grants and the
|
||||
// controller's lease: neither stops the assigner when it is not there (delivery.GrantSource).
|
||||
func sourceOfGrants(name string) bool {
|
||||
return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
|
||||
@@ -170,6 +211,8 @@ var watched = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
buscompose.LeaseBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
@@ -233,7 +276,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
before := w.grantsBucket()
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
if name == buscompose.LeaseBucket && w.grantsBucket() == before {
|
||||
// The controller renews its lease every few seconds: only a change of where grants are read from
|
||||
// makes a pass (delivery.GrantSource).
|
||||
continue
|
||||
}
|
||||
wake()
|
||||
}
|
||||
}
|
||||
@@ -275,7 +324,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && slices.Contains(seatBuckets, name) {
|
||||
if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
@@ -297,13 +346,21 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, len(seatBuckets))
|
||||
placementAppeared := make(chan appeared, len(seatBuckets)+2)
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "the controller's grants are read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
if kvs[a.name] != nil {
|
||||
return nil // bound already, by the other of its two events
|
||||
|
||||
@@ -17,8 +17,10 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
@@ -62,20 +64,30 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
|
||||
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
|
||||
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
lease []byte
|
||||
said string
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
|
||||
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
|
||||
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
@@ -114,13 +126,18 @@ type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
lease []byte // the controller's lease, when it holds one
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
kv := &memKV{values: map[string][]byte{}}
|
||||
if name == buscompose.LeaseBucket && r.lease != nil {
|
||||
kv.values[buscompose.ControllerLeaseKey] = r.lease
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
|
||||
@@ -146,6 +146,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
stale := map[string]neitherOne{}
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774).
|
||||
grants := p.w.grantsBucket()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
@@ -161,7 +163,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
return err == nil && p.w.has(grants, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
@@ -177,8 +179,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
|
||||
if raw := p.w.value(grants, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
@@ -135,6 +136,9 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
if s[state] == nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
@@ -873,6 +877,24 @@ func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules
|
||||
// run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of
|
||||
// the node. A lease that goes back to a controller brings the comparison back.
|
||||
place("n2", "d")
|
||||
assigned("g14", "e")
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`))
|
||||
got, all := p.w.assigned()
|
||||
if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 {
|
||||
t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all)
|
||||
}
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 {
|
||||
t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:])
|
||||
}
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`))
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] {
|
||||
t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
|
||||
@@ -213,8 +213,7 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||
// from their new owner and nothing else changes.
|
||||
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
@@ -222,7 +221,7 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest,
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -524,7 +524,7 @@ func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal,
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded,
|
||||
Seed: prop.Seed, ExportDigest: prop.ExportDigest})
|
||||
Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
@@ -661,7 +661,7 @@ func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.An
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
if p.seat != nil && !assign.Leased && !prop.Seed {
|
||||
if p.seat != nil && !p.w.handedAway() && !prop.Seed {
|
||||
p.seat.callAll(assign.ControllerCalls(before, written, prop.ID, answer.Warrant))
|
||||
}
|
||||
}
|
||||
@@ -729,7 +729,7 @@ func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal
|
||||
now := p.clock()
|
||||
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)})
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()})
|
||||
if len(findings) > 0 {
|
||||
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
|
||||
return
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -214,7 +215,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
@@ -298,7 +299,7 @@ func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)})
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
@@ -308,6 +309,15 @@ func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
|
||||
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
|
||||
// unreadable, is not handed away.
|
||||
func (s *seatTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// handActCall records a hand act in the controller's hand-act log.
|
||||
const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act"
|
||||
|
||||
@@ -315,7 +325,7 @@ const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act"
|
||||
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
|
||||
// records the hand act and says placement-changed; every node returns to the controller's compositions.
|
||||
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
|
||||
if assign.Leased {
|
||||
if s.leased() {
|
||||
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
|
||||
}
|
||||
why := arg(args, "why")
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -142,6 +143,15 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
|
||||
t.Errorf("hand-back = %+v, %v", got, err)
|
||||
}
|
||||
// Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and
|
||||
// there is no controller to hand placement back to.
|
||||
st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}}
|
||||
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
|
||||
t.Errorf("a hold after the lease write: %v", err)
|
||||
}
|
||||
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
|
||||
t.Errorf("a hand-back after the lease write: %v", err)
|
||||
}
|
||||
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
|
||||
@@ -195,21 +195,38 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
|
||||
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
byAssignment, all := w.assignedByController()
|
||||
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
|
||||
// the lease write's own check refuses that order (ADR 0547 §11).
|
||||
placed, seeded := w.placed()
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded)
|
||||
byAssignment, all := w.assignedByController()
|
||||
if away := w.handedAway(); away && seeded {
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, away)
|
||||
return both, map[string]assignment{}
|
||||
}
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, false)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name.
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
|
||||
// the controller's lease is handed away, since the placement is then the one source.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
byAssignment, _ := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded)
|
||||
byAssignment, _ := w.assignedByController()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
|
||||
return differ
|
||||
}
|
||||
|
||||
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
|
||||
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
|
||||
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
|
||||
func (w *world) handedAway() bool {
|
||||
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
|
||||
return away
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
@@ -546,6 +563,16 @@ func (w *world) value(bucket, key string) []byte {
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// grantsBucket is the bucket the grants are read from now (delivery.GrantSource): the grants part's while the controller's
|
||||
// lease is handed away, as the lease bucket's key says it (buscompose.LeaseHandedAway), the controller's in every other
|
||||
// case, a lease not held here included.
|
||||
func (w *world) grantsBucket() string {
|
||||
raw := w.value(buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, raw != nil, nil)
|
||||
bucket, _ := delivery.GrantSource(away)
|
||||
return bucket
|
||||
}
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -125,4 +127,40 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
// Issue 774: grants are read from the grants part's bucket only while the controller's lease is handed away: a grant
|
||||
// only the part holds lets a node's put through then, and with a controller's lease (or none, or an unreadable one) it
|
||||
// holds it, a grant only the controller holds being the one that counts instead.
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
|
||||
for _, row := range []struct {
|
||||
name, lease string
|
||||
inPart, inController bool
|
||||
bucket string
|
||||
granted bool
|
||||
}{
|
||||
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, delivery.PartGrantsBucket, true},
|
||||
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, delivery.PartGrantsBucket, false},
|
||||
{"a controller holds it, only the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, delivery.ControllerGrantsBucket, false},
|
||||
{"a controller holds it, its own grant", `{"instance":"novox/1"}`, false, true, delivery.ControllerGrantsBucket, true},
|
||||
{"no lease read", ``, true, false, delivery.ControllerGrantsBucket, false},
|
||||
{"an unreadable lease", `[1]`, true, false, delivery.ControllerGrantsBucket, false},
|
||||
} {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
if row.lease != "" {
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
|
||||
}
|
||||
if row.inPart {
|
||||
p.w.put(delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
if row.inController {
|
||||
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != row.bucket {
|
||||
t.Errorf("%s: grants are read from %s, want %s", row.name, got, row.bucket)
|
||||
}
|
||||
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
|
||||
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028222736-d747c1128e42
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff h1:EHuM+ed+/M9woMZsMou6/2xm6hIdb1YCFINprrP54Ys=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028222736-d747c1128e42 h1:gKwdXvaJKCfRvWsfj9abfQVPf88bS0n2kAy7Fcuhxe0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028222736-d747c1128e42/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,6 +2,7 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -15,7 +16,8 @@ import (
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
|
||||
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
|
||||
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
|
||||
// is written the controller's assignments are the one source, as before.
|
||||
// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease
|
||||
// handed away) the placement is the one source.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
@@ -32,8 +34,17 @@ const (
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs.
|
||||
func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) {
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's
|
||||
// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the
|
||||
// assignments are then a source of nothing.
|
||||
func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) {
|
||||
if handedAway && seeded {
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if !seeded {
|
||||
return assigned, nil
|
||||
}
|
||||
|
||||
@@ -15,10 +15,6 @@ import (
|
||||
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
|
||||
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
|
||||
|
||||
// Leased says whether the lease write of ADR 0547 has happened, after which the controller's assignments are read no
|
||||
// more (§8). Not before L3: until then the holder keeps the controller in step and refuses what its verbs cannot take.
|
||||
var Leased = false
|
||||
|
||||
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
@@ -32,6 +28,10 @@ type ProposeIn struct {
|
||||
Recorded func(node string) bool
|
||||
Seed bool
|
||||
ExportDigest string
|
||||
// Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk
|
||||
// buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then
|
||||
// the holder keeps the controller in step and refuses what its verbs cannot take.
|
||||
Leased bool
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
@@ -58,7 +58,7 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
|
||||
continue
|
||||
}
|
||||
if !Leased {
|
||||
if !in.Leased {
|
||||
if why := c.BeforeLease(); why != "" {
|
||||
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
|
||||
}
|
||||
|
||||
@@ -60,18 +60,18 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
|
||||
t.Errorf("config values before the lease write: %v", f)
|
||||
}
|
||||
Leased = true
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
leased := in(c)
|
||||
leased.Leased = true
|
||||
if _, _, f := Propose(leased); len(f) != 0 {
|
||||
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
Leased = false
|
||||
|
||||
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
|
||||
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
|
||||
|
||||
@@ -33,9 +33,9 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
|
||||
@@ -24,12 +24,14 @@
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-controller.lease",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-grants.grants",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
|
||||
@@ -153,11 +153,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
||||
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// placement is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
watch.put = gate.Toolchain
|
||||
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
|
||||
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -254,7 +254,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-assignErrs:
|
||||
case err := <-placementErrs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
|
||||
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
|
||||
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
|
||||
// issue 774).
|
||||
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
package main
|
||||
|
||||
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||
// toolchain agreement this file repairs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||
// passed on changed.
|
||||
func (g *toolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||
if err != nil {
|
||||
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
package main
|
||||
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||
key, value := gateReport(t, node, version)
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||
}
|
||||
// ace reports again while unassigned (an old value seen again): still left out.
|
||||
key, value := gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||
t.Fatal("deleting g14's report passed nothing on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef h1:ggwHXCvAOhBCKAY+ErehKEhu/IQTS4poJBRDokGYnDc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -92,8 +92,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
||||
"mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
|
||||
@@ -170,6 +170,21 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
|
||||
const Seat = "mesh-delivery-checker"
|
||||
|
||||
// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try,
|
||||
// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them.
|
||||
func SeatTryTools(tr Trying) []stdio.Tool {
|
||||
verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"}
|
||||
var out []stdio.Tool
|
||||
for _, t := range TryTools(tr) {
|
||||
t.Name = Seat + "." + verbs[t.Name]
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Tools are the read tools over the checker's states.
|
||||
func Tools(r check.Reader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
@@ -308,7 +323,7 @@ func main() {
|
||||
_, err := verdicts.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
tools := append(append(Tools(r), TryTools(trying)...), RecoveryTools(r, written)...)
|
||||
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
if err := stdio.Serve("", logged(tools)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -88,6 +88,24 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike.
|
||||
verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" {
|
||||
t.Fatalf("verbs = %+v", verbs)
|
||||
}
|
||||
again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
if err != nil || again.(map[string]any)["id"] != id {
|
||||
t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id)
|
||||
}
|
||||
if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) {
|
||||
t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw)
|
||||
}
|
||||
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("the verb tried a repository nothing checks")
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
|
||||
@@ -208,11 +208,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
|
||||
// The toolchains first, so a check that starts from the first candidate finds the nodes.
|
||||
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// placement is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
watch.put = gate.Toolchain
|
||||
assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger)
|
||||
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, ch.NodesChanged, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -362,7 +362,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-assignErrs:
|
||||
case err := <-placementErrs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
|
||||
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
|
||||
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
|
||||
// issue 774).
|
||||
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
package main
|
||||
|
||||
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||
// toolchain agreement this file repairs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||
// passed on changed.
|
||||
func (g *toolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||
if err != nil {
|
||||
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package main
|
||||
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||
key, value := gateReport(t, node, version)
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||
}
|
||||
// ace reports again while unassigned (an old value seen again): still left out.
|
||||
key, value := gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||
t.Fatal("deleting g14's report passed nothing on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef h1:ggwHXCvAOhBCKAY+ErehKEhu/IQTS4poJBRDokGYnDc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -38,6 +38,11 @@ type stream struct {
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Streams []stream `json:"streams"`
|
||||
Tools []string `json:"tools"`
|
||||
Module string `json:"module"`
|
||||
@@ -96,8 +101,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
||||
"mesh-delivery-assigner.compositions", "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
@@ -129,7 +134,12 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-delivery-checker" || m.Claims[0].Scope != "mesh" ||
|
||||
!reflect.DeepEqual(m.Claims[0].Serves, []string{"try", "try-read", "try-end"}) {
|
||||
t.Errorf("claims = %+v, want the mesh seat mesh-delivery-checker serving try, try-read and try-end", m.Claims)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the checker has none", k)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,17 @@
|
||||
"module": "mesh-delivery-checker",
|
||||
"version": "1",
|
||||
"slug": "checker",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-delivery-checker",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"try",
|
||||
"try-read",
|
||||
"try-end"
|
||||
]
|
||||
}
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported"
|
||||
@@ -23,8 +34,8 @@
|
||||
"mesh-build-go.toolchains",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.compositions",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-gate.turns",
|
||||
|
||||
@@ -578,23 +578,39 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerCountsOnlyTheAssignedBuildNodesToolchains (issue 1940): the toolchains bucket holds the report of n2, where
|
||||
// mesh-build-go was unassigned (its assignment ended) at another version than n1's, as ace's v26.12.0 stayed beside the
|
||||
// v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's toolchain
|
||||
// alone; when the controller assigns mesh-build-go to n2 again, its report counts and the builders disagree (the guard
|
||||
// stays); when the assignment ends again, the fingerprint comes back. No restart, no timer: the assignments' watch.
|
||||
func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
|
||||
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
|
||||
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
|
||||
// toolchain alone; when the placement puts mesh-build-go on n2 again, its report written since counts and the builders
|
||||
// disagree (the guard stays); when it is taken off again, the fingerprint comes back. No restart, no timer: the
|
||||
// placement's watch (ADR 0549 §3), not the controller's assignments.
|
||||
func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
var assignments jetstream.KeyValue
|
||||
var placement, toolchains jetstream.KeyValue
|
||||
place := func(ctx context.Context, t *testing.T, node string, modules ...string) {
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
value, err := delivery.EncodePlacement(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := placement.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
toolchains = kv
|
||||
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
|
||||
key, _ := delivery.ToolchainKey(node, "go")
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()})
|
||||
@@ -605,18 +621,12 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The controller's assignments mirror, in its layout (mesh-controller internal/chain Assignment).
|
||||
if assignments, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_assignments", History: 1}); err != nil {
|
||||
// The seat mesh-placement's placement, as the assigner writes it (ADR 0567): n1 places mesh-build-go, n2 nothing.
|
||||
if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: delivery.PlacementHistory}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, value := range map[string]string{
|
||||
"n1.mesh-build-go": `{"node":"n1","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z"}`,
|
||||
"n2.mesh-build-go": `{"node":"n2","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z","ended":"2026-10-23T08:00:00Z"}`,
|
||||
} {
|
||||
if _, err := assignments.Put(ctx, key, []byte(value)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
place(ctx, t, "n1", "mesh-build-go")
|
||||
place(ctx, t, "n2")
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
@@ -647,16 +657,22 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||
if !agreed(held) {
|
||||
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
|
||||
}
|
||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
|
||||
place(ctx, t, "n2", "mesh-build-go")
|
||||
key, _ := delivery.ToolchainKey("n2", "go")
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: time.Now().Add(time.Second)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
|
||||
if _, err := toolchains.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// n2's report, written since its placement, counts: the builders disagree. A fingerprint already computed is kept
|
||||
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if e, err := modules.Get(ctx, "avahi"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
|
||||
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
|
||||
t.Fatalf("after n2's placement avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
|
||||
}
|
||||
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
|
||||
if moved == string(data) {
|
||||
@@ -669,12 +685,10 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
||||
}
|
||||
if m := read(disagree); !disagree(m) {
|
||||
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||
}
|
||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
t.Fatalf("within 5 s of a merge after n2's placement avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||
}
|
||||
place(ctx, t, "n2")
|
||||
if m := read(agreed); !agreed(m) {
|
||||
t.Fatalf("within 5 s of n2's unassignment avahi has fp2 %q, none2 %q", m.FP2, m.None2)
|
||||
t.Fatalf("within 5 s of n2's placement taken away avahi has fp2 %q, none2 %q", m.FP2, m.None2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -321,14 +322,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
}
|
||||
}
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
gate := newToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
|
||||
assignErrs, err := watchAssignments(ctx, js, gate, func(ctx context.Context) {
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// placement is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
|
||||
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, func(ctx context.Context) {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
logger.Printf("writing the modules again after an assignment of a build module changed: %v", err)
|
||||
logger.Printf("writing the modules again after the placement of a build module changed: %v", err)
|
||||
}
|
||||
}, logger)
|
||||
}, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -442,7 +443,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
case err := <-sourceErrs:
|
||||
return err
|
||||
case err := <-assignErrs:
|
||||
case err := <-placementErrs:
|
||||
return err
|
||||
case err := <-tools.errs:
|
||||
return err
|
||||
|
||||
@@ -1,211 +0,0 @@
|
||||
package main
|
||||
|
||||
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||
// toolchain agreement this file repairs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||
// passed on changed.
|
||||
func (g *toolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||
if err != nil {
|
||||
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package main
|
||||
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||
key, value := gateReport(t, node, version)
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||
}
|
||||
// ace reports again while unassigned (an old value seen again): still left out.
|
||||
key, value := gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||
t.Fatal("deleting g14's report passed nothing on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-assigner.placement", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
|
||||
@@ -22,8 +22,8 @@
|
||||
"reads": [
|
||||
"mesh-build-go.toolchains",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-checker.previews",
|
||||
"mesh-delivery-gate.source"
|
||||
],
|
||||
|
||||
@@ -19,7 +19,11 @@ import (
|
||||
func serve(t *testing.T) string {
|
||||
t.Helper()
|
||||
m := view.New()
|
||||
m.Apply(view.Entry{Bucket: view.AssignmentsBucket, Key: "a.x", Value: []byte(`{"node":"a","module":"x","build":"b1"}`)})
|
||||
placed, err := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "x"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Apply(view.Entry{Bucket: delivery.PlacementBucket, Key: "placement.a", Value: placed})
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -166,6 +166,15 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Haves(node, module)
|
||||
}},
|
||||
{Name: view.TrustsReadTool,
|
||||
Description: "What every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2): each node's record's trusts " +
|
||||
"(from node-engine_have <node>.node), whether it names the vault's signing key as mesh-vault publishes it in its " +
|
||||
"state signers and whether it still names the controller's, the nodes that do not name the vault's yet (not_yet), " +
|
||||
"and every: true once all do, the gate before the controller's lease is handed away. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return r.Trusts()
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,8 +232,9 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")),
|
||||
ReachState: read(stdio.State("reach")),
|
||||
HeardState: read(stdio.State("mesh-delivery-indexer.heard"))}
|
||||
ReachState: read(stdio.State("reach")),
|
||||
HeardState: read(stdio.State("mesh-delivery-indexer.heard")),
|
||||
SignersState: read(stdio.State("mesh-vault.signers"))}
|
||||
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
|
||||
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
)
|
||||
|
||||
@@ -438,6 +442,41 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
|
||||
t.Errorf("haves of ace (the node's own key too) = %v", got)
|
||||
}
|
||||
|
||||
// L3 s6b (ADR 0550 §2): node_trusts answers for every node in one call whether its record names the vault's key the
|
||||
// vault publishes, and every is true only once all do.
|
||||
vaultPub, _, _ := ed25519.GenerateKey(nil)
|
||||
vaultKey, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now()))
|
||||
vaultFP := delivery.SignerFingerprint(vaultPub)
|
||||
record := func(node string, trusts ...string) string {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{V: delivery.Version, Record: &delivery.NodeRecord{Node: node, Trusts: trusts}})
|
||||
return string(raw)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
haves fake
|
||||
every bool
|
||||
notYet []string
|
||||
}{
|
||||
{"one node trusts the controller's alone", fake{"ace.node": record("ace", "controller key sha256:aa", "vault key "+vaultFP),
|
||||
"g14.node": record("g14", "controller key sha256:aa"), "ace.zsh": `{"fp":"x"}`}, false, []string{"g14"}},
|
||||
{"every node trusts the vault's key", fake{"ace.node": record("ace", "vault key "+vaultFP),
|
||||
"g14.node": record("g14", "controller key sha256:aa", "vault key "+vaultFP)}, true, []string{}},
|
||||
} {
|
||||
tr := view.Reader{HavesState: row.haves.reader().State, SignersState: fake{"key": string(vaultKey)}.reader().State}
|
||||
out, err := run(t, tr, view.TrustsReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
a := out.(view.TrustsAnswer)
|
||||
if a.Every != row.every || !reflect.DeepEqual(a.NotYet, row.notYet) || a.VaultKey != vaultFP || len(a.Nodes) != 2 {
|
||||
t.Errorf("%s: %+v", row.name, a)
|
||||
}
|
||||
}
|
||||
if out, err := run(t, view.Reader{HavesState: fake{"ace.node": record("ace", "vault key "+vaultFP)}.reader().State,
|
||||
SignersState: fake{}.reader().State}, view.TrustsReadTool, map[string]any{}); err != nil || out.(view.TrustsAnswer).Every {
|
||||
t.Errorf("with no key published: %+v, %v", out, err)
|
||||
}
|
||||
|
||||
// More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list
|
||||
// nothing; the whole answer reads MaxStateReads values in parallel and says it was cut.
|
||||
slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) })
|
||||
|
||||
@@ -48,7 +48,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -68,9 +68,20 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8
|
||||
}
|
||||
return b
|
||||
}
|
||||
put(view.AssignmentsBucket, "a.x", []byte(`{"node":"a","module":"x","build":"b1"}`))
|
||||
put(view.AssignmentsBucket, "a.y", []byte(`{"node":"a","module":"y","build":"b2"}`))
|
||||
put(view.AssignmentsBucket, "b.x", []byte(`{"node":"b","module":"x","build":"b1","ended":"2026-10-18T11:30:00Z"}`))
|
||||
// The seat's placement (issue 774, ADR 0549 §3): x and y on a, nothing on b.
|
||||
placeValue := func(node string, modules ...string) []byte {
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
v, err := delivery.EncodePlacement(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
put(delivery.PlacementBucket, "placement.a", placeValue("a", "x", "y"))
|
||||
put(delivery.PlacementBucket, "placement.b", placeValue("b"))
|
||||
put(view.BuildsBucket, "x.build.b1", []byte(`{"id":"b1","module":"x"}`))
|
||||
put(view.BuildsBucket, "x.newest", []byte("b1"))
|
||||
put(delivery.HaveBucket, "a.x", haveValue("b1", delivery.PhaseChecking, delivery.HealthWaits))
|
||||
@@ -150,7 +161,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -236,7 +247,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Command mesh-delivery-view is the operator's view of delivery (novox/hq ADR 0429 §3): it watches the have, assignments
|
||||
// Command mesh-delivery-view is the operator's view of delivery (novox/hq ADR 0429 §3): it watches the have, the seat's placement
|
||||
// and builds buckets on its own bus credential, derives one row per module and node in memory, and serves them on a
|
||||
// local socket (for mesh-delivery-cli) and on a local web page. It starts, stops or retries nothing; the one thing it writes is the operator's answer to an ask that needs no proof of who
|
||||
// answered, under `<ask key>.view` in its bucket answers (ADR 0528 clause 6).
|
||||
@@ -106,7 +106,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
})
|
||||
model.SetBacklog(view.BacklogOver(js, 5*time.Second))
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} {
|
||||
for _, bucket := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} {
|
||||
go follow(ctx, js, bucket, false, model, logger)
|
||||
}
|
||||
// The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556).
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -44,6 +44,8 @@ type Reader struct {
|
||||
ReachState StateRead // the view's own reach (ADR 0556)
|
||||
// HeardState is the FORGE messages the indexer handled (issue 1126 note 76); unset reads none.
|
||||
HeardState StateRead
|
||||
// SignersState is the vault's public signing halves (ADR 0550 §2); unset reads none.
|
||||
SignersState StateRead
|
||||
}
|
||||
|
||||
// ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556).
|
||||
@@ -649,3 +651,89 @@ func (r Reader) Differences(node, module string) (KeptAnswer, error) {
|
||||
func (r Reader) Haves(node, module string) (KeptAnswer, error) {
|
||||
return r.kept(r.HavesState, "the haves", node, module)
|
||||
}
|
||||
|
||||
// TrustsReadTool reads what every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2; design 47's L3
|
||||
// s6b): the gate before the lease write is that every node's record names the vault's key.
|
||||
const TrustsReadTool = "node_trusts"
|
||||
|
||||
// NodeTrusts is what one node's record says it trusts: its lines (delivery.Trusts) and whether they name the vault's key
|
||||
// the vault publishes and the controller's key.
|
||||
type NodeTrusts struct {
|
||||
Trusts []string `json:"trusts"`
|
||||
TrustsVault bool `json:"trusts_vault"`
|
||||
TrustsController bool `json:"trusts_controller"`
|
||||
}
|
||||
|
||||
// TrustsAnswer is node_trusts' answer: the vault's key as it publishes it, each node's trusts, the nodes whose record does
|
||||
// not name the vault's key yet, and whether every node does.
|
||||
type TrustsAnswer struct {
|
||||
VaultKey string `json:"vault_key,omitempty"`
|
||||
Nodes map[string]NodeTrusts `json:"nodes"`
|
||||
NotYet []string `json:"not_yet"`
|
||||
Every bool `json:"every"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// Trusts reads every node's record (`<node>.node` of the haves) and the vault's `signers` key, and answers which nodes
|
||||
// trust the vault's key. Read-only.
|
||||
func (r Reader) Trusts() (TrustsAnswer, error) {
|
||||
if r.HavesState.Get == nil || r.HavesState.Keys == nil {
|
||||
return TrustsAnswer{}, fmt.Errorf("the haves cannot be read here")
|
||||
}
|
||||
out := TrustsAnswer{Nodes: map[string]NodeTrusts{}, NotYet: []string{}}
|
||||
if r.SignersState.Get != nil {
|
||||
raw, err := r.SignersState.Get(delivery.SignersKeyKey)
|
||||
if err != nil {
|
||||
return TrustsAnswer{}, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
k, err := delivery.DecodeSignerKey(raw)
|
||||
if err != nil {
|
||||
return TrustsAnswer{}, err
|
||||
}
|
||||
out.VaultKey = k.Fingerprint
|
||||
}
|
||||
}
|
||||
keys, err := r.HavesState.Keys()
|
||||
if err != nil {
|
||||
return TrustsAnswer{}, err
|
||||
}
|
||||
var records []string
|
||||
for _, k := range keys {
|
||||
if _, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode {
|
||||
records = append(records, k)
|
||||
}
|
||||
}
|
||||
values, _, err := r.HavesState.read(records)
|
||||
if err != nil {
|
||||
return TrustsAnswer{}, err
|
||||
}
|
||||
for key, raw := range values {
|
||||
node := strings.TrimSuffix(key, "."+delivery.HaveNodeWord)
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
out.Unreadable = append(out.Unreadable, key)
|
||||
continue
|
||||
}
|
||||
var t NodeTrusts
|
||||
if h.Record != nil {
|
||||
t.Trusts = h.Record.Trusts
|
||||
}
|
||||
for _, line := range t.Trusts {
|
||||
t.TrustsController = t.TrustsController || strings.HasPrefix(line, delivery.SignerController+" key ")
|
||||
}
|
||||
t.TrustsVault = out.VaultKey != "" && delivery.TrustsKey(t.Trusts, delivery.SignerVault, out.VaultKey)
|
||||
out.Nodes[node] = t
|
||||
if !t.TrustsVault {
|
||||
out.NotYet = append(out.NotYet, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.NotYet)
|
||||
sort.Strings(out.Unreadable)
|
||||
out.Every = out.VaultKey != "" && len(out.Nodes) > 0 && len(out.NotYet) == 0 && len(out.Unreadable) == 0
|
||||
if out.VaultKey == "" {
|
||||
out.Note = "the vault publishes no signing key yet (" + delivery.SignersRead + "), so no node can trust it"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ package view
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -215,7 +216,7 @@ type UnreadableRow struct {
|
||||
// Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values.
|
||||
type Model struct {
|
||||
mu sync.Mutex
|
||||
assigns map[string]assignment
|
||||
placed *delivery.Placed // the seat's placement (the assigner's state, ADR 0567, ADR 0549 §3)
|
||||
builds map[string]buildEntry // by <module>.build.<id>
|
||||
groups map[string]groupEntry // the indexer's candidates, by group
|
||||
modules map[string]moduleEntry // the indexer's modules, by module
|
||||
@@ -263,22 +264,20 @@ type stuckEntry struct {
|
||||
err string
|
||||
}
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the view reads.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Commit string `json:"commit"`
|
||||
Since time.Time `json:"since"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
PulledIn bool `json:"pulled-in"`
|
||||
RequiredBy string `json:"required-by"`
|
||||
}
|
||||
// pair is a module the seat's placement puts on a node.
|
||||
type pair struct{ Node, Module string }
|
||||
|
||||
// open says whether the value is an assignment that stands for the module on its node, as the controller's
|
||||
// Assignment.Stands does. A module pulled in by another stands too.
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
// placedPairs is every module the seat's placement puts on a node, by node then module. The caller holds mu.
|
||||
func (m *Model) placedPairs() []pair {
|
||||
byNode := m.placed.ByNode()
|
||||
var out []pair
|
||||
for _, node := range slices.Sorted(maps.Keys(byNode)) {
|
||||
for _, module := range byNode[node] {
|
||||
out = append(out, pair{Node: node, Module: module})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// build holds only the fields of the controller's Build record the view reads. A record with no state and no failure
|
||||
// is a done build.
|
||||
@@ -356,7 +355,7 @@ type nodeEntry struct {
|
||||
|
||||
// New is an empty model.
|
||||
func New() *Model {
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{},
|
||||
return &Model{placed: delivery.NewPlaced(), builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{},
|
||||
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
||||
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{},
|
||||
modHist: map[string][]delivery.Module{}, srcHist: map[uint64]map[string]string{}, taken: map[string]delivery.Taken{},
|
||||
@@ -529,16 +528,9 @@ func (m *Model) apply(e Entry) {
|
||||
}
|
||||
st, err := delivery.DecodeStuck(e.Value)
|
||||
m.stuck[e.Key] = stuckEntry{value: st, err: errText(err)}
|
||||
case AssignmentsBucket:
|
||||
if e.Deleted {
|
||||
delete(m.assigns, e.Key)
|
||||
return
|
||||
}
|
||||
var a assignment
|
||||
if json.Unmarshal(e.Value, &a) != nil || a.Node == "" || a.Module == "" {
|
||||
return
|
||||
}
|
||||
m.assigns[a.Node+"."+a.Module] = a
|
||||
case delivery.PlacementBucket:
|
||||
// A key that does not read places nothing on its node; the seat's verb placement says why.
|
||||
_, _, _ = m.placed.Put(e.Key, e.Value, e.Deleted, e.At)
|
||||
case delivery.CandidatesBucket:
|
||||
if e.Deleted {
|
||||
delete(m.groups, e.Key)
|
||||
@@ -714,18 +706,12 @@ func (m *Model) snapshot() Snapshot {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
rows := []Row{}
|
||||
for key, a := range m.assigns {
|
||||
if !a.open() {
|
||||
continue
|
||||
}
|
||||
r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned, Assigned: a.Build}
|
||||
if a.PulledIn {
|
||||
r.RequiredBy = a.RequiredBy
|
||||
}
|
||||
for _, a := range m.placedPairs() {
|
||||
r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned}
|
||||
var waits []string
|
||||
hk, herr := delivery.HaveKey(a.Node, a.Module)
|
||||
if herr != nil {
|
||||
hk = key
|
||||
continue
|
||||
}
|
||||
h, has := m.haves[hk]
|
||||
// The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node.
|
||||
@@ -745,17 +731,8 @@ func (m *Model) snapshot() Snapshot {
|
||||
r.Have = h.have.FP
|
||||
}
|
||||
r.Since, r.Detail = h.at, h.have.Detail
|
||||
if wanted {
|
||||
if h.have.FP != w.value.FP {
|
||||
waits = append(waits, "the node has not applied "+w.value.FP)
|
||||
}
|
||||
} else if a.Build != "" && h.have.Build != a.Build {
|
||||
waits = append(waits, "the node has not applied "+a.Build)
|
||||
}
|
||||
}
|
||||
if id := m.newest[a.Module]; id != "" && a.Build != "" && id != a.Build {
|
||||
if b, ok := m.builds[a.Module+".build."+id]; ok && b.done() {
|
||||
waits = append(waits, a.Module+" has a newer build "+id+" not yet assigned")
|
||||
if wanted && h.have.FP != w.value.FP {
|
||||
waits = append(waits, "the node has not applied "+w.value.FP)
|
||||
}
|
||||
}
|
||||
r.Waits = strings.Join(waits, "; ")
|
||||
@@ -766,7 +743,7 @@ func (m *Model) snapshot() Snapshot {
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := m.assigns[node+"."+module]; ok && a.open() {
|
||||
if _, ok := m.placed.Since(node, module); ok {
|
||||
continue
|
||||
}
|
||||
r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err}
|
||||
@@ -984,8 +961,7 @@ func (s Snapshot) Counted() int {
|
||||
|
||||
// The buckets the view reads besides the have bucket (which mesh-sdk's delivery package names).
|
||||
const (
|
||||
AssignmentsBucket = "mesh-controller_assignments"
|
||||
BuildsBucket = "mesh-controller_builds"
|
||||
BuildsBucket = "mesh-controller_builds"
|
||||
)
|
||||
|
||||
// The statuses of a row.
|
||||
@@ -1176,13 +1152,10 @@ func (m *Model) sourceRow() *SourceRow {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP)))
|
||||
}
|
||||
}
|
||||
for key, a := range m.assigns {
|
||||
if !a.open() {
|
||||
continue
|
||||
}
|
||||
for _, a := range m.placedPairs() {
|
||||
wk, werr := delivery.HaveKey(a.Node, a.Module)
|
||||
if werr != nil {
|
||||
wk = key
|
||||
continue
|
||||
}
|
||||
if _, ok := m.wants[wk]; ok {
|
||||
continue
|
||||
|
||||
@@ -25,9 +25,22 @@ func haveE(t *testing.T, node, module string, v []byte, at time.Time) Entry {
|
||||
return Entry{Bucket: delivery.HaveBucket, Key: node + "." + module, Value: v, At: at}
|
||||
}
|
||||
|
||||
func assign(node, module, build, extra string) Entry {
|
||||
v := `{"node":"` + node + `","module":"` + module + `","build":"` + build + `","commit":"c","since":"2026-10-18T11:00:00Z"` + extra + `}`
|
||||
return Entry{Bucket: AssignmentsBucket, Key: node + "." + module, Value: []byte(v), At: t0}
|
||||
// place is the seat's placement of node, as the assigner keeps it (ADR 0567): the modules placed there.
|
||||
func place(t *testing.T, node string, modules ...string) Entry {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodePlacement(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.PlacementBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func buildRec(module, id, extra string) []Entry {
|
||||
@@ -58,7 +71,7 @@ func fill(m *Model, es ...Entry) {
|
||||
// base: two nodes (a, b), three modules (x, y, z).
|
||||
func base(t *testing.T) []Entry {
|
||||
es := []Entry{
|
||||
assign("a", "x", "b1", ""), assign("a", "y", "b2", ""), assign("b", "x", "b1", ""), assign("b", "z", "b3", ""),
|
||||
place(t, "a", "x", "y"), place(t, "b", "x", "z"),
|
||||
haveE(t, "a", "x", have(t, "b1", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(1*time.Minute)),
|
||||
haveE(t, "b", "x", have(t, "b1", delivery.PhaseChecking, delivery.HealthWaits, "starting"), t0.Add(2*time.Minute)),
|
||||
haveE(t, "b", "z", have(t, "b3", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(3*time.Minute)),
|
||||
@@ -77,68 +90,35 @@ func TestRowsPerAssignedModuleAndNode(t *testing.T) { // 1134/AC1
|
||||
t.Fatalf("want 4 rows, got %d counted %d: %+v", len(s.Rows), s.Counted(), s.Rows)
|
||||
}
|
||||
r := find(t, s, "b", "x")
|
||||
want := Row{Node: "b", Module: "x", Status: StatusAssigned, Assigned: "b1", Have: "b1", Phase: "checking", Health: "waits",
|
||||
want := Row{Node: "b", Module: "x", Status: StatusAssigned, Have: "b1", Phase: "checking", Health: "waits",
|
||||
Since: t0.Add(2 * time.Minute), Detail: "starting"}
|
||||
if !reflect.DeepEqual(r, want) {
|
||||
t.Fatalf("row = %+v, want %+v", r, want)
|
||||
}
|
||||
y := find(t, s, "a", "y")
|
||||
if y.Waits != "no have yet" || y.Assigned != "b2" || y.Have != "" {
|
||||
if y.Waits != "no have yet" || y.Have != "" {
|
||||
t.Fatalf("a.y = %+v, want 'no have yet'", y)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitsForTheNodeAndForANewerBuild(t *testing.T) { // 1134/AC2
|
||||
func TestLeftBehindIsWhatThePlacementDoesNotPut(t *testing.T) { // 1134/AC3, issue 774 (the seat's placement)
|
||||
m := New()
|
||||
fill(m, base(t)...)
|
||||
fill(m, haveE(t, "a", "x", have(t, "b0", delivery.PhaseRunning, delivery.HealthOK, ""), t0))
|
||||
if w := find(t, m.Snapshot(), "a", "x").Waits; w != "the node has not applied b1" {
|
||||
t.Fatalf("waits = %q", w)
|
||||
}
|
||||
fill(m, buildRec("z", "b4", "")...)
|
||||
if w := find(t, m.Snapshot(), "b", "z").Waits; w != "z has a newer build b4 not yet assigned" {
|
||||
t.Fatalf("waits = %q", w)
|
||||
}
|
||||
// a newest build that is not done is not a newer build to assign
|
||||
fill(m, buildRec("z", "b5", `,"failed":"boom"`)...)
|
||||
if w := find(t, m.Snapshot(), "b", "z").Waits; w != "" {
|
||||
t.Fatalf("a failed newest build must not be waited on, waits = %q", w)
|
||||
}
|
||||
fill(m, buildRec("z", "b6", `,"state":"asked"`)...)
|
||||
if w := find(t, m.Snapshot(), "b", "z").Waits; w != "" {
|
||||
t.Fatalf("an asked newest build must not be waited on, waits = %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeftBehindIsNotRunningAndPulledInIs(t *testing.T) { // 1134/AC3 (both halves)
|
||||
for name, extra := range map[string]string{"ended": `,"ended":"2026-10-18T11:30:00Z"`, "state": `,"state":"pending"`} {
|
||||
m := New()
|
||||
fill(m, base(t)...)
|
||||
fill(m, assign("b", "z", "b3", extra))
|
||||
s := m.Snapshot()
|
||||
r := find(t, s, "b", "z")
|
||||
if r.Status != StatusLeftBehind {
|
||||
t.Errorf("%s: status = %q, want %q", name, r.Status, StatusLeftBehind)
|
||||
}
|
||||
if s.Counted() != 3 {
|
||||
t.Errorf("%s: counted = %d, want 3", name, s.Counted())
|
||||
}
|
||||
}
|
||||
m := New()
|
||||
fill(m, base(t)...)
|
||||
fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // no assignment at all
|
||||
fill(m, place(t, "b", "x")) // z taken off b
|
||||
s := m.Snapshot()
|
||||
if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 {
|
||||
t.Fatalf("a have with no assignment: %+v counted %d", find(t, s, "a", "w"), s.Counted())
|
||||
if r := find(t, s, "b", "z"); r.Status != StatusLeftBehind || s.Counted() != 3 {
|
||||
t.Errorf("status = %q counted %d, want %q and 3", r.Status, s.Counted(), StatusLeftBehind)
|
||||
}
|
||||
// A module pulled in by another stands: shown and counted, with what required it.
|
||||
m = New()
|
||||
fill(m, base(t)...)
|
||||
fill(m, assign("b", "z", "b3", `,"pulled-in":true,"required-by":"x"`))
|
||||
fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // placed nowhere
|
||||
s = m.Snapshot()
|
||||
r := find(t, s, "b", "z")
|
||||
if r.Status != StatusAssigned || r.RequiredBy != "x" || r.Have != "b3" || s.Counted() != 4 {
|
||||
t.Fatalf("pulled-in row = %+v counted %d, want shown and counted with required-by x", r, s.Counted())
|
||||
if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 {
|
||||
t.Fatalf("a have with no placement: %+v counted %d", find(t, s, "a", "w"), s.Counted())
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.PlacementBucket, Key: "placement.b", Deleted: true})
|
||||
if s := m.Snapshot(); s.Counted() != 2 || find(t, s, "b", "x").Status != StatusLeftBehind {
|
||||
t.Fatalf("b's placement deleted: counted %d, %+v", s.Counted(), s.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,18 +161,13 @@ func TestOnlyTheAffectedRowChanges(t *testing.T) { // 1134/AC5
|
||||
if r := find(t, after, "b", "x"); r.Phase != "running" || r.Health != "ok" || !r.Since.Equal(t0.Add(9*time.Minute)) {
|
||||
t.Fatalf("changed row = %+v", r)
|
||||
}
|
||||
// an assignment change and a build change touch only their rows
|
||||
fill(m, assign("a", "y", "b7", ""))
|
||||
if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != "b7" {
|
||||
t.Fatalf("assignment change not seen: %+v", r)
|
||||
// a want's change touches only its row
|
||||
fill(m, wantE(t, "a", "y", delivery.Want{FP: fpNew}))
|
||||
if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != fpNew {
|
||||
t.Fatalf("want change not seen: %+v", r)
|
||||
}
|
||||
if !reflect.DeepEqual(find(t, m.Snapshot(), "b", "z"), find(t, after, "b", "z")) {
|
||||
t.Fatal("an assignment of another row changed b.z")
|
||||
}
|
||||
fill(m, buildRec("x", "b8", "")...)
|
||||
s := m.Snapshot()
|
||||
if find(t, s, "a", "x").Waits == "" || find(t, s, "b", "x").Waits == "" || find(t, s, "b", "z").Waits != "" {
|
||||
t.Fatalf("a build of x must touch x's rows only: %+v", s.Rows)
|
||||
t.Fatal("a want of another row changed b.z")
|
||||
}
|
||||
// a deleted key (a bucket's expiry) removes its value
|
||||
m.Apply(Entry{Bucket: delivery.HaveBucket, Key: "a.x", Deleted: true})
|
||||
|
||||
@@ -101,7 +101,7 @@ func TestEveryWantIsARowAgainstItsHave(t *testing.T) {
|
||||
func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew),
|
||||
assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""),
|
||||
place(t, "ace", "web"), place(t, "g14", "web"),
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
|
||||
@@ -119,7 +119,7 @@ func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) {
|
||||
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") {
|
||||
t.Errorf("a want that is not the module's fingerprint: %+v", s)
|
||||
}
|
||||
fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", ""))
|
||||
fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), place(t, "novox", "web"))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") {
|
||||
t.Errorf("a node assigned the module with no want: %+v", s)
|
||||
@@ -177,7 +177,7 @@ func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) {
|
||||
said = append(said, e)
|
||||
return nil
|
||||
})
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""),
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), place(t, "ace", "web"),
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v before any have", said)
|
||||
@@ -213,7 +213,7 @@ func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) {
|
||||
// have: the row shows the fingerprint wanted and waits while the node runs another.
|
||||
func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
fill(m, place(t, "ace", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
r := find(t, m.Snapshot(), "ace", "web")
|
||||
if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") {
|
||||
t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r)
|
||||
@@ -228,10 +228,12 @@ func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) {
|
||||
// and a have nothing is assigned to keeps the module's name.
|
||||
func TestADottedModuleNameIsFoundAndKept(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK),
|
||||
// The seat's placement refuses a dotted module name (mesh-sdk delivery.Placement), so such a module is never placed:
|
||||
// its have is a row left behind under its own name.
|
||||
fill(m, place(t, "ace"), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK),
|
||||
haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK))
|
||||
s := m.Snapshot()
|
||||
if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" {
|
||||
if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Status != StatusLeftBehind {
|
||||
t.Errorf("the dotted module's row = %+v", r)
|
||||
}
|
||||
if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind {
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-vault.signers", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
@@ -119,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
}
|
||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"assigner_want",
|
||||
"assigner_differences",
|
||||
"node_haves",
|
||||
"node_trusts",
|
||||
"build_usage",
|
||||
"view_reach",
|
||||
"indexer_module_history",
|
||||
@@ -30,8 +31,6 @@
|
||||
{ "name": "reach", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-build-go.pauses",
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-image.pauses",
|
||||
@@ -39,9 +38,11 @@
|
||||
"mesh-build-node.pauses",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-delivery-assigner.asks",
|
||||
"mesh-delivery-assigner.differences",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-checker.tries",
|
||||
@@ -54,6 +55,7 @@
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-delivery-watchman.stuck",
|
||||
"mesh-vault.signers",
|
||||
"node-engine.have"
|
||||
],
|
||||
"own-secrets": {
|
||||
|
||||
@@ -214,6 +214,10 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
return s.c.State(state).Put(key, json.RawMessage(value))
|
||||
}
|
||||
|
||||
func (s ownStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() }
|
||||
|
||||
func (s ownStore) Delete(state, key string) error { return s.c.State(state).Delete(key) }
|
||||
|
||||
// emitting is the process connection as the part's emitter: its events, with no options.
|
||||
type emitting struct{ c *process.Conn }
|
||||
|
||||
|
||||
@@ -18,6 +18,41 @@ type creating struct {
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
// CreateConsumer creates one durable consumer an account needs, by its stream and name (ADR 0561 §6): the create alone;
|
||||
// one the bus holds answers ErrHeld and is left as it is.
|
||||
func (c creating) CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
defer cancel()
|
||||
if _, err := c.js.Consumer(ctx, stream, cfg.Durable); err == nil {
|
||||
return grants.ErrHeld
|
||||
}
|
||||
_, err := c.js.CreateConsumer(ctx, stream, cfg)
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, jetstream.ErrConsumerExists):
|
||||
return grants.ErrHeld
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return fmt.Errorf("no answer in %s: a create the bus refuses is answered by none (%w)", requestWithin, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// MatchConsumer brings a held consumer to cfg on a person's ask (grants_consumer_match), keeping where it delivers from:
|
||||
// the server refuses a changed deliver policy, and a consumer moved would replay or lose what it holds.
|
||||
func (c creating) MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
defer cancel()
|
||||
held, err := c.js.Consumer(ctx, stream, cfg.Durable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
have := held.CachedInfo().Config
|
||||
cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = have.DeliverPolicy, have.OptStartSeq, have.OptStartTime
|
||||
_, err = c.js.UpdateConsumer(ctx, stream, cfg)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c creating) CreateDeclared(cfg jetstream.StreamConfig) error {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
defer cancel()
|
||||
|
||||
@@ -62,14 +62,9 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// keptStates is what the part keeps, each readable (ADR 0495 §2-§3): the view and a session read them.
|
||||
func keptStates() []announce.State {
|
||||
var out []announce.State
|
||||
for _, s := range grants.States {
|
||||
out = append(out, announce.State{Name: s, Readable: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
// keptStates is what the part keeps (ADR 0495 §2-§3): each readable, for the view and a session, but the wanted accounts,
|
||||
// by the vault alone (ADR 0561 §2).
|
||||
func keptStates() []announce.State { return grants.Announced() }
|
||||
|
||||
// run is the process: it returns when ctx ends, or when it has no credential or cannot dial the bus at all. A bucket or
|
||||
// read the bus refuses never ends it.
|
||||
@@ -117,8 +112,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
bus := newBusReader(ctx, js)
|
||||
f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn},
|
||||
Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js},
|
||||
Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf}
|
||||
Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js},
|
||||
Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf,
|
||||
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
|
||||
f.Watch = bus
|
||||
// The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on
|
||||
// the warrant once (mesh-sdk asker), after its refusals.
|
||||
@@ -138,8 +134,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
}
|
||||
// The controller makes this module's durable consumer on EVENTS from the manifest's consumes (filtered on the two
|
||||
// condition subjects); it is pulled and every message acknowledged, so its pending never grows. Where it is not there
|
||||
// yet the pass says so and hears again at the next change.
|
||||
// condition subjects and the vault's account-issued; ADR 0561 §6 gives that to the part from the lease write); it is
|
||||
// pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and
|
||||
// hears again at the next change. The vault's bus-accounts is watched beside it, so a hash is composed either way.
|
||||
f.Hear = func(handle func(string, []byte)) error {
|
||||
cons, err := js.Consumer(ctx, "EVENTS", account.Node+"_"+account.Module)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
@@ -39,6 +40,10 @@ func (runtimeStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
return stdio.State(state).Put(key, json.RawMessage(value))
|
||||
}
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
// runtimeBus calls tools and says events through the runtime.
|
||||
type runtimeBus struct{}
|
||||
|
||||
@@ -113,6 +118,43 @@ func toolsOf(p *grants.Part) []stdio.Tool {
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.ToolAccounts, Description: "The bus accounts the mesh wants (novox/hq ADR 0561 §2, §9): every account the vault " +
|
||||
"mints that placement composes (a module's, a process's, a machine's provisioner client's), as the grants part last " +
|
||||
"wrote it to its state accounts, which the vault alone reads: user, kind, node, module, process, the bus's address and " +
|
||||
"fingerprint, a module's claims with the verbs each seat promises, and when the key was written; one user's or all; " +
|
||||
"what the last pass wrote, deleted or kept; and the durable consumers the accounts need as the last pass found them " +
|
||||
"(missing, differing and why, created; whether the lease is handed away). Never a password. Read-only: " +
|
||||
"grants_compose_now writes them again.",
|
||||
Input: schema(map[string]string{"user": "one bus user, e.g. ace.mesh-build-go"}),
|
||||
Run: func(a map[string]any) (any, error) { return p.Accounts(grants.Answer(a, "user")) }},
|
||||
{Name: grants.ToolConsumerMatch, Description: "Bring one durable consumer an account needs (a module's on EVENTS, a process's on " +
|
||||
"its work queue; novox/hq ADR 0561 §6) to the configuration the bus composer composes, by hand: the recovery when " +
|
||||
"the condition mesh-grants.consumer-differs names it. Asks the part's process, which alone may, and keeps where the " +
|
||||
"consumer delivers from. Refused while the controller holds its lease (the consumers are the controller's then) and " +
|
||||
"for a consumer the records do not compose. grants_user_list's last pass says what became of it.",
|
||||
Input: schema(map[string]string{"stream": "the stream, e.g. EVENTS", "consumer": "the consumer, e.g. ace_mesh-build-go"}, "stream", "consumer"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.MatchConsumer(grants.Answer(a, "stream"), grants.Answer(a, "consumer"))
|
||||
}},
|
||||
{Name: grants.ToolAcceptHashes, Description: "Release the hold of a user list that changes several loaded users' hashes at " +
|
||||
"once (novox/hq ADR 0561 §7; the condition mesh-grants.user-list-held names them), by a person's act with why: names " +
|
||||
"exactly the users held, bound to the held list's digest, so the part's process writes that list once at its next pass " +
|
||||
"and says hashes-accepted. It never releases another hold (a failed read, a user dropped for want of a hash, a lost " +
|
||||
"right), and a list composed otherwise since is held again for a new acceptance.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"users": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "the users held, as the condition names them"},
|
||||
"why": map[string]any{"type": "string", "description": "why their hashes change"}}, "required": []string{"users", "why"}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
var users []string
|
||||
if list, ok := a["users"].([]any); ok {
|
||||
for _, u := range list {
|
||||
if s, ok := u.(string); ok {
|
||||
users = append(users, strings.TrimSpace(s))
|
||||
}
|
||||
}
|
||||
}
|
||||
return p.AcceptHashes(stdio.CallerOf(a), users, grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.ToolTakeOverState, Description: "Where the switch stands: the list the bus node follows, the ask open now, the request, and " +
|
||||
"every act with what became of it (asked, switched, declined, expired, refused and why).",
|
||||
Input: schema(nil),
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-grants
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c h1:ngcItoLuh2uP24toZvkCXGA/3+TBrLTwTbE28XpYDWY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -57,6 +58,45 @@ type fakeBus struct {
|
||||
refuseUsers bool
|
||||
// rootFree answers the controller's root-free verb: nil is no answerer at all (no responders).
|
||||
rootFree func(machines []string) (any, error)
|
||||
// consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and
|
||||
// consumerCreates each consumer the part asked to create, `<stream>.<name>`.
|
||||
consumers map[string][]map[string]any
|
||||
consumerCreates []string
|
||||
}
|
||||
|
||||
// MatchConsumer is the bus's update of a held durable consumer to the configuration given.
|
||||
func (b *fakeBus) MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.consumerCreates = append(b.consumerCreates, "match "+stream+"."+cfg.Durable)
|
||||
for _, c := range b.consumers[stream] {
|
||||
if c["name"] == cfg.Durable {
|
||||
c["max_deliver"] = cfg.MaxDeliver
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New("consumer not found")
|
||||
}
|
||||
|
||||
// CreateConsumer is the bus's create of a durable consumer: held when the bus holds one of that name, else held from now.
|
||||
func (b *fakeBus) CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.consumerCreates = append(b.consumerCreates, stream+"."+cfg.Durable)
|
||||
for _, c := range b.consumers[stream] {
|
||||
if c["name"] == cfg.Durable {
|
||||
return grants.ErrHeld
|
||||
}
|
||||
}
|
||||
filters := append([]string{}, cfg.FilterSubjects...)
|
||||
if cfg.FilterSubject != "" {
|
||||
filters = []string{cfg.FilterSubject}
|
||||
}
|
||||
if b.consumers == nil {
|
||||
b.consumers = map[string][]map[string]any{}
|
||||
}
|
||||
b.consumers[stream] = append(b.consumers[stream], map[string]any{"name": cfg.Durable, "filters": filters, "max_deliver": cfg.MaxDeliver})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) LastMembership(node, module string) ([]byte, uint64, bool, error) {
|
||||
@@ -177,6 +217,9 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
}
|
||||
return json.Marshal(map[string]any{"users": out})
|
||||
case grants.CallStreams:
|
||||
if stream, _ := args["stream"].(string); stream != "" {
|
||||
return json.Marshal(map[string]any{"detail": []map[string]any{{"name": stream, "consumer_detail": b.consumers[stream]}}})
|
||||
}
|
||||
return json.Marshal(map[string]any{"detail": b.streams})
|
||||
case grants.CallRootFree:
|
||||
if b.rootFree == nil {
|
||||
@@ -217,6 +260,27 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
return b.revs[state+"/"+key], nil
|
||||
}
|
||||
|
||||
func (s ownStore) Keys(state string) ([]string, error) {
|
||||
b := s.b
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
var out []string
|
||||
for k := range b.own[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s ownStore) Delete(state, key string) error {
|
||||
b := s.b
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.writes = append(b.writes, state+" delete")
|
||||
delete(b.own[state], key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Emit(event string, _ any) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
@@ -1144,7 +1208,7 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("membership = %v", ms)
|
||||
}
|
||||
gr, _ := p.Grants("mod-x")
|
||||
if g, _ := gr.(map[string]any); g["writes"] != grants.WritesNoGrant || g["count"] != 1 {
|
||||
if g, _ := gr.(map[string]any); g["writes"] != grants.WritesGrants || g["controller_count"] != 1 {
|
||||
t.Errorf("grants = %v", gr)
|
||||
}
|
||||
|
||||
@@ -1371,7 +1435,12 @@ func TestGrants(t *testing.T) {
|
||||
var fresh []asks.Condition
|
||||
fp := partOn(b)
|
||||
fp.Objects = p.Objects
|
||||
fp.Asker, fp.Say = fa, func(c asks.Condition) error { fresh = append(fresh, c); return nil }
|
||||
fp.Asker, fp.Say = fa, func(c asks.Condition) error {
|
||||
if c.Key != grants.GrantMissingCondition { // its own row below
|
||||
fresh = append(fresh, c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fp.Compose()
|
||||
if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != asks.StateCleared {
|
||||
t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh)
|
||||
@@ -1504,7 +1573,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, w := range b.writes {
|
||||
if !slices.Contains(grants.States, w) {
|
||||
if !slices.Contains(grants.States, strings.TrimSuffix(w, " delete")) {
|
||||
t.Errorf("wrote %s, which is not the part's own state", w)
|
||||
}
|
||||
}
|
||||
@@ -1714,6 +1783,31 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// 774 (ADR 0563 §2, §3): the node-engine's manifest (mesh-host) is placed on every node though no node is assigned it,
|
||||
// so its declared state acts is created as a placed module's, and composed for the user list's create right.
|
||||
t.Run("assertions: the node-engine's declared state is created though no node is assigned it", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.manifestOf(buscompose.EngineModule, `{"module":"mesh-host","version":"1","state":[{"name":"acts"}]}`)
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
p.Create = b
|
||||
p.Compose()
|
||||
if !slices.Contains(b.creates, "KV_"+buscompose.EngineActsBucket) {
|
||||
t.Fatalf("the node-engine's acts were not created: %v", b.creates)
|
||||
}
|
||||
recs, _ := grants.Records(grants.ReadInputs(b, &grants.Manifests{}))
|
||||
if recs.Engine == nil || len(recs.Engine.State) != 1 || recs.Engine.State[0].Bucket() != buscompose.EngineActsBucket {
|
||||
t.Fatalf("the records carry the node-engine as %+v", recs.Engine)
|
||||
}
|
||||
for node, mods := range recs.Assigned {
|
||||
for _, d := range mods {
|
||||
if d.Module == buscompose.EngineModule {
|
||||
t.Errorf("the node-engine is a module of %s, which would give it a module's account", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("assertions: a refused create is said, never stops it, and is tried at the next event", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
@@ -1823,6 +1917,463 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("the refusal was not said: %v", logged)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.<module>.<fp> in shadow, once each and only for a
|
||||
// module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no
|
||||
// account with a hash and every read was made, and the tool says every difference from the controller's.
|
||||
t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
fp := "src2:" + strings.Repeat("ab", 32)
|
||||
key := func(module string) string { k, _ := delivery.PartGrantKey(module, fp); return k }
|
||||
hash := func(user string) {
|
||||
b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1})
|
||||
}
|
||||
held := func() []string { return slices.Sorted(maps.Keys(b.own[grants.StateGrants])) }
|
||||
// The controller's grant for this fingerprint, beside the stale one the fixture holds for another.
|
||||
delete(b.states[grants.ReadControllerGrant], "mod-x.0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123")
|
||||
b.put(grants.ReadControllerGrant, key("mod-x"), delivery.ControllerGrant{V: delivery.Version, Accounts: []string{"a.mod-x"}})
|
||||
hash("a.mod-x")
|
||||
hash("a.nats")
|
||||
res := p.Compose()
|
||||
if res.Grants == nil || !slices.Equal(held(), []string{key("mod-x"), key("nats")}) || len(res.Grants.Written) != 2 {
|
||||
t.Fatalf("grants of the hashed modules: held %v, %+v", held(), res.Grants)
|
||||
}
|
||||
if g, err := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key("mod-x")]); err != nil || !slices.Equal(g.Accounts, []string{"a.mod-x"}) {
|
||||
t.Errorf("the grant of mod-x is %+v, %v", g, err)
|
||||
}
|
||||
said := func() int {
|
||||
n := 0
|
||||
for _, e := range b.events {
|
||||
if e == grants.EventGrantWritten {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if said() != 2 {
|
||||
t.Errorf("grant-written said %d times for two grants: %v", said(), b.events)
|
||||
}
|
||||
if res := p.Compose(); len(res.Grants.Written) != 0 || said() != 2 {
|
||||
t.Errorf("a second pass wrote %v and said %d", res.Grants.Written, said())
|
||||
}
|
||||
// The tool: equal on mod-x, and nats is the part's alone (the controller's fixture holds none).
|
||||
got, err := p.Grants("")
|
||||
gm, _ := got.(map[string]any)
|
||||
diff, _ := gm["differences"].([]grants.GrantDifference)
|
||||
if err != nil || gm["equal"] != false || len(diff) != 1 || diff[0].Key != key("nats") || diff[0].Kind != "only the part's" {
|
||||
t.Errorf("grants = %+v, %v", got, err)
|
||||
}
|
||||
// A user with a hash later: written once. Without it again: deleted. Unread hashes: nothing deleted.
|
||||
hash("b.mod-y")
|
||||
if res := p.Compose(); !slices.Equal(res.Grants.Written, []string{key("mod-y")}) {
|
||||
t.Errorf("mod-y was granted %v when its account got a hash", res.Grants.Written)
|
||||
}
|
||||
b.refused[grants.ReadAccounts] = true
|
||||
if res := p.Compose(); res.Grants == nil || len(res.Grants.Deleted)+len(res.Grants.Written) != 0 || len(res.Grants.Kept) != 1 || len(held()) != 3 {
|
||||
t.Errorf("a pass that could not read the hashes changed grants: %+v, held %v", res.Grants, held())
|
||||
}
|
||||
delete(b.refused, grants.ReadAccounts)
|
||||
delete(b.states[grants.ReadAccounts], "b.mod-y")
|
||||
if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 {
|
||||
t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held())
|
||||
}
|
||||
})
|
||||
|
||||
// Issue 1126 (C5): a module whose source commit has not moved since a build of the old form, with a done key of the
|
||||
// new form at that commit (link2pay, snake, f1-circuits, txt-game), is granted at the indexer's fingerprint, never
|
||||
// judged by its commit; and a placed module left without a grant is said, never passed over: in the pass, and as the
|
||||
// condition mesh-grants.grant-missing once the part's grants are the ones read, cleared when it has one.
|
||||
t.Run("grants: at the indexer's fingerprint whatever the commit, and a placed module without one is said", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
var said []asks.Condition
|
||||
p.Say = func(c asks.Condition) error {
|
||||
if c.Key == grants.GrantMissingCondition {
|
||||
said = append(said, c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
|
||||
fp := "src2:d4afe7a1abab5ccdbef6d8629b2245006a2aec60290f430f7b4756e2930ed60c"
|
||||
b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(xManifest)),
|
||||
Repository: "novox/Link2Pay", Commit: "5d20e8bfbc953c51e8ebe3c671fe883df9c36e15"})
|
||||
b.put(grants.ReadModules, "mod-y", delivery.Module{V: delivery.Version, None2: "its builder's version is not read yet",
|
||||
Manifest: delivery.ManifestDigest([]byte(yManifest)), Repository: "novox/y", Commit: "c"})
|
||||
for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} {
|
||||
b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1})
|
||||
}
|
||||
res := p.Compose()
|
||||
key, _ := delivery.PartGrantKey("mod-x", fp)
|
||||
if _, held := b.own[grants.StateGrants][key]; !held {
|
||||
t.Errorf("mod-x has no grant at the indexer's fingerprint %s: %+v", fp, res.Grants)
|
||||
}
|
||||
if res.Grants == nil || len(res.Grants.Missing) != 1 || !strings.HasPrefix(res.Grants.Missing[0],
|
||||
"mod-y: no fingerprint of the new form: its builder's version is not read yet") {
|
||||
t.Fatalf("missing = %+v", res.Grants)
|
||||
}
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "mod-y") {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
if p.Compose(); len(said) != 1 {
|
||||
t.Errorf("the same missing grant was said again: %+v", said)
|
||||
}
|
||||
b.put(grants.ReadModules, "mod-y", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(yManifest)),
|
||||
Repository: "novox/y", Commit: "c"})
|
||||
if res := p.Compose(); len(res.Grants.Missing) != 0 || len(said) != 2 || said[1].State != asks.StateCleared {
|
||||
t.Errorf("once mod-y has a grant: missing %v, said %+v", res.Grants.Missing, said)
|
||||
}
|
||||
})
|
||||
|
||||
// The guard of ADR 0549 §6 for this repository, as mesh-catalog's checks/placement is for the catalogue: no module.json
|
||||
// here reads the controller's assignments unless ../assignments-readers names it with why, and that list only shrinks.
|
||||
t.Run("assignments readers: only the named modules read the controller's assignments, and the list only shrinks", func(t *testing.T) {
|
||||
raw, err := os.ReadFile("../assignments-readers")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed := map[string]bool{}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) > 1 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]] = true
|
||||
} else if len(f) == 1 {
|
||||
t.Errorf("../assignments-readers names %s with no why", f[0])
|
||||
}
|
||||
}
|
||||
paths, _ := filepath.Glob("../*/module.json")
|
||||
reads := map[string]bool{}
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Reads []string `json:"reads"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
if slices.Contains(m.Reads, grants.ReadAssignments) {
|
||||
reads[m.Module] = true
|
||||
if !allowed[m.Module] {
|
||||
t.Errorf("%s reads %s: read where modules run from the seat mesh-placement's placement (ADR 0549 §3); the lease write "+
|
||||
"is refused while a module reads the assignments (ADR 0549 §6)", m.Module, grants.ReadAssignments)
|
||||
}
|
||||
}
|
||||
}
|
||||
for m := range allowed {
|
||||
if !reads[m] {
|
||||
t.Errorf("../assignments-readers names %s, which no longer reads %s: take it off the list", m, grants.ReadAssignments)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 774 (the L3 review, ADR 0549 §3, §6): once the controller's lease is handed away the assignments are frozen and place
|
||||
// nothing; the seat's placement alone does. A module left only in the stale assignments loses its account and its
|
||||
// grant names no account of it; before the lease write the two are still unioned.
|
||||
t.Run("placement alone once the lease is handed away: a module only in the stale assignments has no account or grant of it", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-x"}}})
|
||||
b.states[grants.ReadPlacement] = map[string][]byte{"placement.a": pl}
|
||||
for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} {
|
||||
b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1})
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("ab", 32)
|
||||
key, _ := delivery.PartGrantKey("mod-y", fp)
|
||||
grantOf := func() []string {
|
||||
g, _ := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key])
|
||||
return g.Accounts
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-28", "epoch": 60})
|
||||
p.Compose()
|
||||
if b.own[grants.StateAccounts]["b.mod-y"] == nil || !slices.Equal(grantOf(), []string{"b.mod-y"}) {
|
||||
t.Fatalf("under the controller's lease the assignments no longer count: account %v, grant %v",
|
||||
b.own[grants.StateAccounts]["b.mod-y"] != nil, grantOf())
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
|
||||
in := grants.ReadInputs(b, &grants.Manifests{})
|
||||
if len(in.Assigned["b"]) != 0 || !slices.Equal(in.Assigned["a"], []string{"mod-x", "nats"}) || in.BusNode != "a" {
|
||||
t.Errorf("after the lease write the stale assignments still place: %v, bus %q", in.Assigned, in.BusNode)
|
||||
}
|
||||
res := p.Compose()
|
||||
if res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) || slices.Contains(grantOf(), "b.mod-y") {
|
||||
t.Fatalf("a module only in the stale assignments kept its account or grant: %+v, grant %v", res.Accounts, grantOf())
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC4 (novox/hq ADR 0561 §2): every account the vault mints that the records compose is written to the state
|
||||
// accounts once, with the bus's address and a module's claims, read by the vault alone; a key is deleted when placement
|
||||
// no longer names its user, and never in a pass whose reads failed.
|
||||
t.Run("wanted accounts: written once each, read by the vault alone, retired only when placement drops them", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
|
||||
p.Compose()
|
||||
p.Compose()
|
||||
wrote := 0
|
||||
for _, w := range b.writes {
|
||||
if w == grants.StateAccounts {
|
||||
wrote++
|
||||
}
|
||||
}
|
||||
held := slices.Sorted(maps.Keys(b.own[grants.StateAccounts]))
|
||||
if !slices.Equal(held, []string{"a.mod-x", "a.nats", "b.mod-y"}) || wrote != 3 {
|
||||
t.Fatalf("accounts %v written %d times in two passes, want the three modules' once each", held, wrote)
|
||||
}
|
||||
nats, err := buscompose.ReadWantedAccount("a.nats", b.own[grants.StateAccounts]["a.nats"])
|
||||
if err != nil || nats.URL != "nats://a.mesh:4222" || nats.Fingerprint != "ab" || len(nats.Claims) != 1 ||
|
||||
nats.Claims[0].Seat != catalogue.BrokerSeat || nats.At.IsZero() {
|
||||
t.Errorf("a.nats is wanted as %+v (%v)", nats, err)
|
||||
}
|
||||
var readers []string
|
||||
for _, s := range grants.Announced() {
|
||||
if s.Name == grants.StateAccounts {
|
||||
if s.Readable {
|
||||
t.Error("the wanted accounts are announced readable by anyone")
|
||||
}
|
||||
readers = s.Readers
|
||||
}
|
||||
}
|
||||
if !slices.Equal(readers, []string{"mesh-vault"}) || buscompose.ReaderMay("mesh-delivery-view", buscompose.GrantsAccounts) {
|
||||
t.Errorf("the wanted accounts are read by %v", readers)
|
||||
}
|
||||
// A read that fails retires nothing; once read, the user placement dropped is retired.
|
||||
delete(b.states[grants.ReadAssignments], "b.mod-y")
|
||||
b.refused[grants.ReadResolved] = true
|
||||
if res := p.Compose(); res.Accounts == nil || len(res.Accounts.Deleted) != 0 || b.own[grants.StateAccounts]["b.mod-y"] == nil {
|
||||
t.Fatalf("a pass whose read failed deleted %+v", res.Accounts)
|
||||
}
|
||||
delete(b.refused, grants.ReadResolved)
|
||||
if res := p.Compose(); res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) {
|
||||
t.Fatalf("the user placement dropped is not retired: %+v", res.Accounts)
|
||||
}
|
||||
// The tool reads them, one and all, each with when it was written.
|
||||
one, err := p.Accounts("a.nats")
|
||||
all, err2 := p.Accounts("")
|
||||
if err != nil || err2 != nil || one.(map[string]any)["count"] != 1 || all.(map[string]any)["count"] != 2 ||
|
||||
one.(map[string]any)["accounts"].([]buscompose.WantedAccount)[0].At.IsZero() {
|
||||
t.Errorf("the tool reads %v and %v (%v %v)", one, all, err, err2)
|
||||
}
|
||||
if _, err := p.Accounts("b.mod-y"); err == nil {
|
||||
t.Error("the tool reads an account no longer wanted")
|
||||
}
|
||||
p.Pass()
|
||||
if all, _ := p.Accounts(""); all.(map[string]any)["consumers"] == nil {
|
||||
t.Errorf("the tool does not say the account consumers in shadow: %v", all)
|
||||
}
|
||||
// With no bus address no account is written, and the pass says why.
|
||||
bare := partOn(fixture())
|
||||
if res := bare.Compose(); res.Accounts != nil || !strings.Contains(strings.Join(res.Errors, " "), "no bus address") {
|
||||
t.Errorf("a part with no bus address wrote %+v: %v", res.Accounts, res.Errors)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash
|
||||
// within one pass; an event of another kind does not.
|
||||
t.Run("the vault's account-issued composes the list again", func(t *testing.T) {
|
||||
b := fixture()
|
||||
var handle func(string, []byte)
|
||||
p := partOn(b)
|
||||
f := &grants.Follower{Part: p, Watch: &fakeWatcher{opened: make(chan string, 16)}, Logf: func(string, ...any) {},
|
||||
Hear: func(h func(string, []byte)) error { handle = h; return nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { f.Run(ctx); close(done) }()
|
||||
defer func() { cancel(); <-done }()
|
||||
waitFor(t, func() bool { return b.count(grants.CallUsers) > 0 && handle != nil })
|
||||
held := func() bool {
|
||||
for _, c := range p.Last().UserList.Credentials.Users {
|
||||
if c.User == "a.mod-x" {
|
||||
return c.HashHeld
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
if held() {
|
||||
t.Fatal("a.mod-x has a hash before the vault made one")
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.states[grants.ReadAccounts] = map[string][]byte{"a.mod-x": []byte(`{"user":"a.mod-x","hash":"$2a$10$x","generation":1}`)}
|
||||
b.mu.Unlock()
|
||||
handle(buscompose.AccountIssuedSubject, []byte(`{"user":"a.mod-x","kind":"module","node":"a","module":"mod-x","generation":1}`))
|
||||
waitFor(t, held)
|
||||
if !slices.Contains(grants.Consumes, grants.EventAccountIssued) || !grants.IsAccountIssued(grants.EventAccountIssued) ||
|
||||
grants.IsAccountIssued("mesh.mod.mesh-vault.event.secret-changed") {
|
||||
t.Errorf("the part hears %v", grants.Consumes)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC6 (novox/hq ADR 0561 §6): the durable consumers an account needs, with the controller's configuration, are
|
||||
// compared in shadow while the controller holds the lease, and created, only where the bus lacks them, from the write
|
||||
// that hands the lease away; one the bus holds is never changed.
|
||||
t.Run("account consumers: compared while the controller holds the lease, created once it is handed away", func(t *testing.T) {
|
||||
const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done"]}`
|
||||
b := fixture()
|
||||
b.manifestOf("mod-c", cManifest)
|
||||
b.assign("b", "mod-c")
|
||||
b.loaded = shadowOf(t, b)
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-27", "epoch": 57})
|
||||
p := partOn(b)
|
||||
p.Consumers = b
|
||||
res := p.Compose()
|
||||
if res.Consumers == nil || res.Consumers.On || !slices.Contains(res.Consumers.Missing, "EVENTS.b_mod-c") || len(b.consumerCreates) != 0 {
|
||||
t.Fatalf("while the controller holds the lease: %+v, created %v", res.Consumers, b.consumerCreates)
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
|
||||
b.consumers = map[string][]map[string]any{"EVENTS": {{"name": "b_other", "filters": []string{"x"}, "max_deliver": 5}}}
|
||||
res = p.Compose()
|
||||
if !res.Consumers.On || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c"}) {
|
||||
t.Fatalf("once handed away: %+v", res.Consumers)
|
||||
}
|
||||
var made map[string]any
|
||||
for _, c := range b.consumers["EVENTS"] {
|
||||
if c["name"] == "b_mod-c" {
|
||||
made = c
|
||||
}
|
||||
}
|
||||
if made == nil || made["max_deliver"] != 5 || !slices.Contains(made["filters"].([]string), "mesh.mod.mod-y.event.done") {
|
||||
t.Errorf("b_mod-c was made as %v, not as the controller makes it", made)
|
||||
}
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 1 {
|
||||
t.Errorf("a consumer the bus holds was asked again: %+v, %v", res.Consumers, b.consumerCreates)
|
||||
}
|
||||
// A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to
|
||||
// the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease.
|
||||
var said []asks.Condition
|
||||
p.Say = func(c asks.Condition) error {
|
||||
if c.Key != grants.GrantMissingCondition { // its own row
|
||||
said = append(said, c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
made["max_deliver"] = 9
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Differ) != 1 || !strings.Contains(res.Consumers.Differ[0], "EVENTS.b_mod-c: max deliver 9") ||
|
||||
len(said) != 1 || said[0].Key != grants.ConsumerDiffersCondition || said[0].State != asks.StateRaised ||
|
||||
!strings.Contains(said[0].Explanation, "EVENTS.b_mod-c") {
|
||||
t.Fatalf("a differing consumer: %+v, said %+v", res.Consumers, said)
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/2/2026-10-27", "epoch": 58})
|
||||
if _, err := p.MatchConsumer("EVENTS", "b_mod-c"); err == nil || !strings.Contains(err.Error(), "controller holds the lease") {
|
||||
t.Errorf("matched by hand under the controller's lease: %v", err)
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
|
||||
if _, err := p.MatchConsumer("EVENTS", "b_nothing"); err == nil {
|
||||
t.Error("a consumer the records do not compose was asked to match")
|
||||
}
|
||||
if _, err := p.MatchConsumer("EVENTS", "b_mod-c"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res = p.Compose()
|
||||
if !slices.Contains(b.consumerCreates, "match EVENTS.b_mod-c") || made["max_deliver"] != 5 || len(res.Consumers.Matched) != 1 {
|
||||
t.Errorf("the ask to match: %+v, %v", res.Consumers, b.consumerCreates)
|
||||
}
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Differ) != 0 || said[len(said)-1].State != asks.StateCleared {
|
||||
t.Errorf("once matched: %+v, said %+v", res.Consumers, said)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no
|
||||
// rotation by the vault naming each, is held: the list in force stays, user-list-held is raised naming those users. One
|
||||
// user's change, or changes the vault's rotation names, are written.
|
||||
t.Run("a list changing several loaded users' hashes is held unless the vault's rotation names each", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
hash := func(user, h string, generation int, rotating bool) {
|
||||
v := map[string]any{"user": user, "hash": h, "generation": generation, "fingerprint": fmt.Sprintf("sha256:%d", generation)}
|
||||
if rotating {
|
||||
v["rotating"] = map[string]any{"login": user + "--b", "generation": generation, "revision": 7}
|
||||
}
|
||||
b.put(grants.ReadAccounts, user, v)
|
||||
}
|
||||
for _, u := range shadowOf(t, b) {
|
||||
hash(u.User, "$2a$10$first", 1, false)
|
||||
}
|
||||
busKey, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
|
||||
p := partOn(b)
|
||||
p.Objects = &fakeObjects{}
|
||||
var said []asks.Condition
|
||||
p.Say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
|
||||
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose()
|
||||
inForce, _, _ := p.Sealed()
|
||||
if inForce.Digest == "" {
|
||||
t.Fatal("no list written after take-over")
|
||||
}
|
||||
// One user's hash changes: written.
|
||||
hash("a.mod-x", "$2a$10$second", 2, false)
|
||||
p.Compose()
|
||||
if now, _, _ := p.Sealed(); now.Digest == inForce.Digest {
|
||||
t.Fatal("a list changing one loaded user's hash was not written")
|
||||
}
|
||||
inForce, _, _ = p.Sealed()
|
||||
// Two change at once, no rotation names them: held, said naming both.
|
||||
hash("a.mod-x", "$2a$10$third", 3, false)
|
||||
hash("a.nats", "$2a$10$third", 2, false)
|
||||
said = nil
|
||||
res := p.Compose()
|
||||
if now, _, _ := p.Sealed(); now.Digest != inForce.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x, a.nats") {
|
||||
t.Fatalf("a list changing two loaded users' hashes: %+v", res.Sealed)
|
||||
}
|
||||
if len(said) != 1 || said[0].Key != grants.HeldCondition || said[0].State != asks.StateRaised || !strings.Contains(said[0].Summary, "a.mod-x, a.nats") {
|
||||
t.Errorf("the hold was said as %+v", said)
|
||||
}
|
||||
// The hold is its own guard, beside take-over's refusals (ADR 0562): the switch stays taken over.
|
||||
if p.SwitchState() != grants.TookOver {
|
||||
t.Errorf("the hold moved the switch to %s", p.SwitchState())
|
||||
}
|
||||
// A person accepts exactly the named users' changes of the held list, with why: refused for another set, with no
|
||||
// why, or once the list changed since; accepted, the next pass writes it once, said as a hand act.
|
||||
if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x"}, "rotated by hand"); err == nil || !strings.Contains(err.Error(), "a.mod-x, a.nats") {
|
||||
t.Errorf("an acceptance of another set than the held one: %v", err)
|
||||
}
|
||||
if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x", "a.nats"}, ""); err == nil {
|
||||
t.Error("an acceptance with no why")
|
||||
}
|
||||
if _, err := p.AcceptHashes("person.jochen", []string{"a.nats", "a.mod-x"}, "the vault's store was restored"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The list changed since the acceptance (a.mod-x's hash again): the acceptance is bound to the held list, refused.
|
||||
hash("a.mod-x", "$2a$10$fourth", 4, false)
|
||||
res = p.Compose()
|
||||
if now, _, _ := p.Sealed(); now.Digest != inForce.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") {
|
||||
t.Fatalf("an acceptance of a list that changed since released it: %+v", res.Sealed)
|
||||
}
|
||||
if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x", "a.nats"}, "the vault's store was restored"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.events = nil
|
||||
p.Compose()
|
||||
if now, _, _ := p.Sealed(); now.Digest == inForce.Digest || said[len(said)-1].State != asks.StateCleared ||
|
||||
!slices.Contains(b.events, grants.EventHashesAccepted) {
|
||||
t.Fatalf("an accepted hold was not released once and said: %+v, events %v", said, b.events)
|
||||
}
|
||||
inForce, _, _ = p.Sealed()
|
||||
// The vault's rotation names each: written, the warning cleared.
|
||||
hash("a.mod-x", "$2a$10$fifth", 5, true)
|
||||
hash("a.nats", "$2a$10$fifth", 3, true)
|
||||
p.Compose()
|
||||
if now, _, _ := p.Sealed(); now.Digest == inForce.Digest || said[len(said)-1].State != asks.StateCleared {
|
||||
t.Errorf("changes the vault's rotation names were held: %+v", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type fakeAsker struct {
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// **The accounts the mesh wants, said to the vault** (novox/hq ADR 0561 §2, issue 774). The bus composer derives every
|
||||
// user from placement, the manifests and the wants; for each account the vault mints (a module's, a process's, a
|
||||
// machine's provisioner client's), the part writes one key of its state `accounts`, keyed by the bus user, with the
|
||||
// fields the controller's credential carries beside the password. A key is written only when it differs, and deleted
|
||||
// only when placement no longer names its user and every input of the pass was read: a refused read never retires an
|
||||
// account. The vault alone reads the state (buscompose.NamedReaders); it mints each user it holds no value for.
|
||||
|
||||
// StateAccounts is the part's state of wanted accounts.
|
||||
const StateAccounts = buscompose.AccountsState
|
||||
|
||||
// AccountsReaders are the only modules that may read StateAccounts.
|
||||
var AccountsReaders = buscompose.NamedReaders[buscompose.GrantsAccounts]
|
||||
|
||||
// Announced is what the part says it keeps: every state readable by any reader, but the wanted accounts, by the vault
|
||||
// alone (ADR 0495 §2-§3, ADR 0561 §2).
|
||||
func Announced() []announce.State {
|
||||
var out []announce.State
|
||||
for _, s := range States {
|
||||
if s == StateAccounts {
|
||||
out = append(out, announce.State{Name: s, Readers: slices.Clone(AccountsReaders)})
|
||||
continue
|
||||
}
|
||||
out = append(out, announce.State{Name: s, Readable: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BusAddress is where every node on the private network reaches the bus, and its certificate's fingerprint: the part's
|
||||
// own credential says both, since the controller gave every node's account the bus holder's overlay name (its
|
||||
// brokerReachableAt).
|
||||
type BusAddress struct {
|
||||
URL string
|
||||
Fingerprint string
|
||||
}
|
||||
|
||||
// WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address. A
|
||||
// node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure.
|
||||
func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[string]buscompose.WantedAccount, error) {
|
||||
principals, err := buscompose.Users(recs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]buscompose.WantedAccount{}
|
||||
for _, p := range principals {
|
||||
w := buscompose.WantedAccount{User: p.Username(), Kind: p.Kind, Node: p.Node, Module: p.Module, Process: p.Process,
|
||||
URL: bus.URL, Fingerprint: bus.Fingerprint}
|
||||
switch p.Kind {
|
||||
case buscompose.KindModule, buscompose.KindNodeTools:
|
||||
if m, ok := manifestOn(in, p.Node, p.Module); ok {
|
||||
w.Claims = ClaimsOf(m)
|
||||
}
|
||||
case buscompose.KindProcess, buscompose.KindProvisioner:
|
||||
default:
|
||||
continue
|
||||
}
|
||||
out[w.User] = w
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// manifestOn is the manifest a node runs of module: the build its want names, else the trunk's.
|
||||
func manifestOn(in Inputs, node, module string) (catalogue.Manifest, bool) {
|
||||
if m, ok := in.Running[node+"."+module]; ok {
|
||||
return m, true
|
||||
}
|
||||
m, ok := in.Manifests[module]
|
||||
return m, ok
|
||||
}
|
||||
|
||||
// ClaimsOf is a module's claims as its credential carries them: the controller's claimsFor, over the mesh's own seats
|
||||
// (its seat table is seeded from them): each claim's seat and scope, and the verbs the runtime serves for it, the claim's
|
||||
// own or every verb the seat promises; a seat the mesh does not define keeps the claim's scope and no verbs.
|
||||
func ClaimsOf(m catalogue.Manifest) []buscompose.ClaimedSeat {
|
||||
if len(m.Claims) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := map[string]catalogue.Seat{}
|
||||
for _, s := range catalogue.DefaultSeats() {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
var out []buscompose.ClaimedSeat
|
||||
for _, c := range m.Claims {
|
||||
claimed := buscompose.ClaimedSeat{Seat: c.Name, Scope: c.At()}
|
||||
if s, known := byName[c.Name]; known {
|
||||
claimed.Scope = s.Scope
|
||||
claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)})
|
||||
}
|
||||
out = append(out, claimed)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Accounts answers the tool accounts (ADR 0561 §9): the wanted accounts as the part last wrote them, one user's or all,
|
||||
// each with when it was written; and what the last pass did to them. Never a password: the state holds none.
|
||||
func (p *Part) Accounts(user string) (any, error) {
|
||||
users := []string{user}
|
||||
if user == "" {
|
||||
keys, err := p.Store.Keys(StateAccounts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own %s cannot be listed: %w", StateAccounts, err)
|
||||
}
|
||||
users = keys
|
||||
}
|
||||
out := []buscompose.WantedAccount{}
|
||||
for _, u := range users {
|
||||
raw, _, found, err := p.Store.Get(StateAccounts, u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own %s %s cannot be read: %w", StateAccounts, u, err)
|
||||
}
|
||||
if !found {
|
||||
if user != "" {
|
||||
return nil, fmt.Errorf("no account %s is wanted: placement composes no module, process or provisioner client of that user", user)
|
||||
}
|
||||
continue
|
||||
}
|
||||
w, err := buscompose.ReadWantedAccount(u, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
answer := map[string]any{"accounts": out, "count": len(out), "read_by": AccountsReaders}
|
||||
if raw, _, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
|
||||
var res Result
|
||||
if json.Unmarshal(raw, &res) == nil && res.Accounts != nil {
|
||||
answer["last_pass"] = res.Accounts
|
||||
}
|
||||
// And the durable consumers the accounts need, as the last pass found them: missing and differing ones are read
|
||||
// here in shadow too, where no condition is raised (ADR 0561 §6, §9).
|
||||
if res.Consumers != nil {
|
||||
answer["consumers"] = res.Consumers
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// AccountsWritten is what one pass did to the state of wanted accounts.
|
||||
type AccountsWritten struct {
|
||||
Wanted int `json:"wanted"`
|
||||
Written []string `json:"written,omitempty"`
|
||||
Deleted []string `json:"deleted,omitempty"`
|
||||
// Kept are keys placement no longer names that this pass did not delete, with why.
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
// writeAccounts writes each wanted account whose key differs, and deletes the key of a user no longer placed, but only
|
||||
// when every input of the pass was read.
|
||||
func (p *Part) writeAccounts(res *Result, in Inputs, recs buscompose.Records) {
|
||||
if p.Bus.URL == "" {
|
||||
res.Errors = append(res.Errors, "the wanted accounts are not written: the part knows no bus address to give them (its own credential names none)")
|
||||
return
|
||||
}
|
||||
wanted, err := WantedAccounts(in, recs, p.Bus)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, "the wanted accounts could not be composed: "+err.Error())
|
||||
return
|
||||
}
|
||||
done := &AccountsWritten{Wanted: len(wanted)}
|
||||
res.Accounts = done
|
||||
for _, user := range sortedKeys(wanted) {
|
||||
w := wanted[user]
|
||||
if raw, _, found, err := p.Store.Get(StateAccounts, user); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateAccounts, user, err))
|
||||
continue
|
||||
} else if found {
|
||||
if was, err := buscompose.ReadWantedAccount(user, raw); err == nil && was.Same(w) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
w.At = p.now()
|
||||
data, _ := json.Marshal(w)
|
||||
if _, err := p.Store.Put(StateAccounts, user, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateAccounts, user, err))
|
||||
continue
|
||||
}
|
||||
done.Written = append(done.Written, user)
|
||||
}
|
||||
held, err := p.Store.Keys(StateAccounts)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no key of a user gone is deleted: %v", StateAccounts, err))
|
||||
return
|
||||
}
|
||||
failed := in.ReadFailures()
|
||||
placed := Placed(in)
|
||||
for _, user := range held {
|
||||
if _, still := wanted[user]; still {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case placed(user):
|
||||
done.Kept = append(done.Kept, user+": placement still names it (composed with no account this pass)")
|
||||
continue
|
||||
case len(failed) > 0:
|
||||
done.Kept = append(done.Kept, user+": a read of this pass failed, so no account is retired")
|
||||
continue
|
||||
}
|
||||
if err := p.Store.Delete(StateAccounts, user); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateAccounts, user, err))
|
||||
continue
|
||||
}
|
||||
done.Deleted = append(done.Deleted, user)
|
||||
}
|
||||
sort.Strings(done.Kept)
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774). The controller creates a module's
|
||||
// consumer on EVENTS and each process's on its work queue when it issues the account. From the write that hands its
|
||||
// lease away the grants part creates them, with the configuration the bus composer derives (buscompose.AccountConsumers),
|
||||
// only where the bus lacks one; one the bus holds is never changed or deleted, a difference is only said. Before that
|
||||
// write they are compared in shadow alone, so two parts never create one consumer.
|
||||
|
||||
// ReadLease is the controller's lease as the part reads it (buscompose.NamedReaders names the vault and the part), and
|
||||
// LeaseKey its one key.
|
||||
const (
|
||||
ReadLease = buscompose.ControllerLease
|
||||
LeaseKey = buscompose.ControllerLeaseKey
|
||||
)
|
||||
|
||||
// ConsumerCreator creates one durable consumer the bus does not hold: the part's process alone, whose account is granted
|
||||
// the create of each account's consumer by stream and name. One the bus holds answers ErrHeld.
|
||||
type ConsumerCreator interface {
|
||||
CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error
|
||||
// MatchConsumer brings a held consumer to cfg, keeping where it delivers from: only on a person's ask
|
||||
// (grants_consumer_match), never by a pass.
|
||||
MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error
|
||||
}
|
||||
|
||||
// ConsumersState is what one pass found and did of the account consumers.
|
||||
type ConsumersState struct {
|
||||
// On says the lease is handed away, so the part creates what the bus lacks; Why says why it is on or off.
|
||||
On bool `json:"on"`
|
||||
Why string `json:"why"`
|
||||
// Wanted is how many the records compose; Missing those the bus lacks (`<stream>.<name>`), Differ those it holds
|
||||
// otherwise, each with how.
|
||||
Wanted int `json:"wanted"`
|
||||
Missing []string `json:"missing,omitempty"`
|
||||
Differ []string `json:"differ,omitempty"`
|
||||
Created []string `json:"created,omitempty"`
|
||||
// Matched is a consumer brought to the composed configuration on a person's ask, or why not.
|
||||
Matched []string `json:"matched,omitempty"`
|
||||
NotCreated []string `json:"not_created,omitempty"`
|
||||
// Unread is a stream whose consumers could not be read: nothing on it is compared or created.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// heldConsumer is one consumer as the bus module's nats_streams reads it for one stream.
|
||||
type heldConsumer struct {
|
||||
Name string `json:"name"`
|
||||
Filters []string `json:"filters"`
|
||||
MaxDeliver int `json:"max_deliver"`
|
||||
}
|
||||
|
||||
// heldConsumers reads the consumers the bus holds on stream, by name.
|
||||
func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
|
||||
raw, err := c.Call(CallStreams, map[string]any{"stream": stream})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s: %w", CallStreams, stream, err)
|
||||
}
|
||||
var got struct {
|
||||
Detail []struct {
|
||||
Name string `json:"name"`
|
||||
Consumers []heldConsumer `json:"consumer_detail"`
|
||||
} `json:"detail"`
|
||||
}
|
||||
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
|
||||
return nil, fmt.Errorf("%s %s answered no consumers: %w", CallStreams, stream, err)
|
||||
}
|
||||
out := map[string]heldConsumer{}
|
||||
for _, d := range got.Detail {
|
||||
if d.Name != stream {
|
||||
continue
|
||||
}
|
||||
for _, c := range d.Consumers {
|
||||
out[c.Name] = c
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// leaseHandedAway reads the lease as the part's reads give it.
|
||||
func leaseHandedAway(r Reader) (bool, string) {
|
||||
raw, found, err := r.Get(ReadLease, LeaseKey)
|
||||
return buscompose.LeaseHandedAway(raw, found, err)
|
||||
}
|
||||
|
||||
// consumers compares the account consumers the records compose with those the bus holds, and creates the missing ones
|
||||
// once the lease is handed away and the part holds a creator.
|
||||
func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
users, err := buscompose.Users(recs)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, "the account consumers could not be composed: "+err.Error())
|
||||
return
|
||||
}
|
||||
st := &ConsumersState{}
|
||||
st.On, st.Why = leaseHandedAway(p.Read)
|
||||
res.Consumers = st
|
||||
wanted := buscompose.AccountConsumers(users)
|
||||
st.Wanted = len(wanted)
|
||||
byStream := map[string][]buscompose.Consumer{}
|
||||
for _, mc := range wanted {
|
||||
byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer)
|
||||
}
|
||||
for _, stream := range sortedKeys(byStream) {
|
||||
held, err := heldConsumers(p.Call, stream)
|
||||
if err != nil {
|
||||
st.Unread = append(st.Unread, stream+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
for _, c := range byStream[stream] {
|
||||
key := stream + "." + c.Name
|
||||
h, ok := held[c.Name]
|
||||
if ok {
|
||||
why := consumerDiffers(c, h)
|
||||
if why != "" && key == match && st.On && p.Consumers != nil {
|
||||
if err := p.Consumers.MatchConsumer(stream, buscompose.ConsumerConfig(c)); err != nil {
|
||||
st.Matched = append(st.Matched, key+": not matched: "+err.Error())
|
||||
} else {
|
||||
st.Matched = append(st.Matched, key+": matched, asked by hand")
|
||||
continue
|
||||
}
|
||||
}
|
||||
if why != "" {
|
||||
st.Differ = append(st.Differ, key+": "+why)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !st.On || p.Consumers == nil {
|
||||
st.Missing = append(st.Missing, key)
|
||||
continue
|
||||
}
|
||||
switch err := p.Consumers.CreateConsumer(stream, buscompose.ConsumerConfig(c)); {
|
||||
case err == nil:
|
||||
st.Created = append(st.Created, key)
|
||||
p.say(res, EventAsserted, map[string]any{"consumer": c.Name, "stream": stream})
|
||||
case errors.Is(err, ErrHeld):
|
||||
default:
|
||||
st.Missing = append(st.Missing, key)
|
||||
st.NotCreated = append(st.NotCreated, key+": "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
p.sayDiffer(st)
|
||||
}
|
||||
|
||||
// sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the
|
||||
// lease write) differs from its composed configuration, naming each with the fields that differ, and clears it when they
|
||||
// match. Said once per set of differences, again when it changes. Before the lease write the consumers are the
|
||||
// controller's, and a difference is only said in the pass.
|
||||
func (p *Part) sayDiffer(st *ConsumersState) {
|
||||
if p.Say == nil {
|
||||
return
|
||||
}
|
||||
why := ""
|
||||
if st.On && len(st.Differ) > 0 {
|
||||
why = strings.Join(st.Differ, "; ")
|
||||
}
|
||||
p.holdMu.Lock()
|
||||
defer p.holdMu.Unlock()
|
||||
if p.differKnown && why == p.differSaid {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: ConsumerDiffersCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
|
||||
if why == "" {
|
||||
c.State = asks.StateCleared
|
||||
} else {
|
||||
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d durable consumers differ from their composed configuration", len(st.Differ))
|
||||
c.Headline, c.Needs = "A module's consumer is not as composed", "a look, and grants_consumer_match for each"
|
||||
c.Explanation = asks.NeedsYou + "a look. The bus holds these consumers otherwise than the bus composer composes them, " +
|
||||
"and the grants part never changes a held one by itself (its delivery position would be lost): " + why
|
||||
c.Resolved = "every account consumer matches, by grants_consumer_match or otherwise"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
p.differSaid, p.differKnown = why, true
|
||||
}
|
||||
}
|
||||
|
||||
// ConsumerDiffersCondition is the key of the warning raised while an account consumer differs (ADR 0561 §6).
|
||||
const ConsumerDiffersCondition = "mesh-grants.consumer-differs"
|
||||
|
||||
// MatchConsumer is the tool consumer-match (ADR 0561 §6, §9): it asks the part's process to bring one account consumer
|
||||
// the records compose to its composed configuration, keeping where it delivers from. Refused while the controller holds
|
||||
// the lease (the consumers are the controller's then), and for a consumer the records do not compose.
|
||||
func (p *Part) MatchConsumer(stream, name string) (any, error) {
|
||||
if stream == "" || name == "" {
|
||||
return nil, errors.New("consumer-match names a stream and a consumer")
|
||||
}
|
||||
if on, why := leaseHandedAway(p.Read); !on {
|
||||
return nil, fmt.Errorf("not asked: %s; the controller makes and matches the consumers while it holds the lease", why)
|
||||
}
|
||||
in := ReadInputs(p.Read, &p.cache)
|
||||
recs, _ := Records(in)
|
||||
users, err := buscompose.Users(recs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
found := false
|
||||
for _, mc := range buscompose.AccountConsumers(users) {
|
||||
if mc.Consumer.Stream == stream && mc.Consumer.Name == name {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the records compose no account consumer %s on %s: the part brings only those to their configuration", name, stream)
|
||||
}
|
||||
ask, _ := json.Marshal(Ask{By: "grants_consumer_match", At: p.now(), Consumer: stream + "." + name})
|
||||
rev, err := p.Store.Put(StateUserList, KeyAsked, ask)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written: %w", err)
|
||||
}
|
||||
return map[string]any{"asked": rev, "says": "the part's process brings " + stream + "." + name + " to its composed " +
|
||||
"configuration at this ask, keeping where it delivers from; the last pass's consumers.matched says what became of it"}, nil
|
||||
}
|
||||
|
||||
// consumerDiffers says how a held consumer differs from the composed one in what nats_streams reads back, or "".
|
||||
func consumerDiffers(c buscompose.Consumer, h heldConsumer) string {
|
||||
var why []string
|
||||
want, have := slices.Sorted(slices.Values(c.Filters)), slices.Sorted(slices.Values(h.Filters))
|
||||
if !slices.Equal(want, have) {
|
||||
why = append(why, fmt.Sprintf("filters %v, composed %v", have, want))
|
||||
}
|
||||
maxDeliver := c.MaxDeliver
|
||||
if maxDeliver == 0 {
|
||||
maxDeliver = -1
|
||||
}
|
||||
if h.MaxDeliver != maxDeliver {
|
||||
why = append(why, fmt.Sprintf("max deliver %d, composed %d", h.MaxDeliver, maxDeliver))
|
||||
}
|
||||
sort.Strings(why)
|
||||
return strings.Join(why, "; ")
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -18,7 +19,7 @@ type Watcher interface {
|
||||
// Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
|
||||
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
|
||||
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
|
||||
var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement}
|
||||
var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease}
|
||||
|
||||
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
|
||||
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
|
||||
@@ -28,9 +29,9 @@ type Follower struct {
|
||||
Logf func(string, ...any)
|
||||
// Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none.
|
||||
Asked func(wake func()) error
|
||||
// Hear hears the controller seat's condition events (Consumes) and hands each, by event name, with its body, to
|
||||
// handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: a machine's
|
||||
// root-free word changed.
|
||||
// Hear hears the controller seat's condition events and the vault's account-issued (Consumes) and hands each, by event
|
||||
// name, with its body, to handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass:
|
||||
// a machine's root-free word changed; and so does an account the vault minted (ADR 0561 §1), whose hash the list carries.
|
||||
Hear func(handle func(event string, body []byte)) error
|
||||
|
||||
wake chan struct{}
|
||||
@@ -46,6 +47,8 @@ type Follower struct {
|
||||
held string
|
||||
notAsserted string
|
||||
refusedMans string
|
||||
// leaseOn is whether the lease was handed away at its last change heard, once leaseSeen.
|
||||
leaseOn, leaseSeen bool
|
||||
}
|
||||
|
||||
func (f *Follower) init() {
|
||||
@@ -85,6 +88,15 @@ func (f *Follower) logf(format string, args ...any) {
|
||||
// heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the
|
||||
// bus module's loaded list moving.
|
||||
func (f *Follower) heard(state, key string, value []byte) bool {
|
||||
if state == ReadLease {
|
||||
// Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6).
|
||||
on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
was, seen := f.leaseOn, f.leaseSeen
|
||||
f.leaseOn, f.leaseSeen = on, true
|
||||
return !seen || was != on
|
||||
}
|
||||
if state != ReadHave {
|
||||
return true
|
||||
}
|
||||
@@ -124,7 +136,7 @@ func (f *Follower) open() {
|
||||
}
|
||||
if f.Hear != nil && !f.heard_ {
|
||||
err := f.Hear(func(event string, body []byte) {
|
||||
if IsRootNotFree(body) {
|
||||
if IsAccountIssued(event) || IsRootNotFree(body) {
|
||||
f.Wake()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The grants the part writes** (novox/hq ADR 0547 §4, §5, ADR 0561 §2, §6; issue 774). A module's grant is the record
|
||||
// that its bus credential exists for the fingerprint the catalogue's trunk names for it: `<module>.<fingerprint hex>` in
|
||||
// the part's state grants, naming the accounts (the module's own and each process's) the vault holds a hash for. It is the
|
||||
// controller's grant as the controller composes it (cmd/mesh-controller grants.go grantsFrom), so the two compare by key.
|
||||
// Until the controller's lease is handed away nobody reads the part's keys: the assigner reads the controller's
|
||||
// (delivery.GrantSource), and the tool grants says every difference, so the switch is made on a measured equality.
|
||||
//
|
||||
// A module of which an account has no hash yet is granted the accounts that have one; a module none of whose accounts has
|
||||
// one has no grant, and one it was written for is deleted (an older grant naming a node whose account is gone would let the
|
||||
// assigner want it, and a grant naming no account names every node). Nothing else is ever deleted, and nothing is written
|
||||
// or deleted in a pass that could not read the vault's hashes, the indexer's modules or a manifest of the module.
|
||||
|
||||
// GrantsWritten is what one pass did to the state grants.
|
||||
type GrantsWritten struct {
|
||||
Wanted int `json:"wanted"`
|
||||
Written []string `json:"written,omitempty"`
|
||||
Deleted []string `json:"deleted,omitempty"`
|
||||
// Kept says why a pass wrote and deleted nothing, or left a module out.
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
// Missing is each module the placement or an assignment puts on a machine that has no grant after the pass,
|
||||
// `<module>: <why>`: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once
|
||||
// the part's grants are the ones read, it is the condition mesh-grants.grant-missing.
|
||||
Missing []string `json:"missing,omitempty"`
|
||||
}
|
||||
|
||||
// GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126).
|
||||
const GrantMissingCondition = "mesh-grants.grant-missing"
|
||||
|
||||
// MissingGrants is each module assigned or placed on a machine that want leaves without a grant, `<module>: <why>`,
|
||||
// sorted: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash
|
||||
// in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits
|
||||
// and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it,
|
||||
// and that is the fingerprint granted). Pure.
|
||||
func MissingGrants(in Inputs, want map[string]delivery.ControllerGrant, drop []string, skip map[string]bool) []string {
|
||||
placed := map[string]bool{}
|
||||
for _, ms := range in.Assigned {
|
||||
for _, m := range ms {
|
||||
placed[m] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, module := range sortedKeys(placed) {
|
||||
fp, has := in.Fingerprints[module]
|
||||
key, err := delivery.PartGrantKey(module, fp)
|
||||
if has && err == nil {
|
||||
if _, granted := want[key]; granted {
|
||||
continue
|
||||
}
|
||||
}
|
||||
why := ""
|
||||
switch {
|
||||
case skip[module]:
|
||||
why = "its manifest at the indexer's trunk is refused"
|
||||
case !has:
|
||||
why = "no fingerprint of the new form: " + orSaid(in.NoFingerprint[module], "the indexer holds no value of it")
|
||||
case err != nil:
|
||||
why = fmt.Sprintf("no grant key for %s: %v", fp, err)
|
||||
case slices.Contains(drop, key):
|
||||
why = "none of its accounts at " + fp + " has a hash in the vault"
|
||||
default:
|
||||
why = "no grant composed at " + fp
|
||||
}
|
||||
out = append(out, module+": "+why)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orSaid(s, none string) string {
|
||||
if s == "" {
|
||||
return none
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// sayMissing raises mesh-grants.grant-missing while a placed module has no grant, once the controller's lease is handed
|
||||
// away (the part's grants are then the ones the assigner reads), and clears it when every one has one; in shadow the
|
||||
// pass and the tool say them. Said once per set, again when it changes, and always at the first pass after a start.
|
||||
func (p *Part) sayMissing(missing []string) {
|
||||
if p.Say == nil {
|
||||
return
|
||||
}
|
||||
why := ""
|
||||
if on, _ := leaseHandedAway(p.Read); on && len(missing) > 0 {
|
||||
why = strings.Join(missing, "; ")
|
||||
}
|
||||
p.holdMu.Lock()
|
||||
defer p.holdMu.Unlock()
|
||||
if p.missingKnown && why == p.missingSaid {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: GrantMissingCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
|
||||
if why == "" {
|
||||
c.State = asks.StateCleared
|
||||
} else {
|
||||
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d placed module(s) have no grant, so no want can name them", len(missing))
|
||||
c.Headline, c.Needs = "A placed module has no bus grant", "a look at why each has none"
|
||||
c.Explanation = asks.NeedsYou + "a look. The grants part wrote no grant for these modules, so the assigner's " +
|
||||
"credential rule gives them no want: " + why
|
||||
c.Resolved = "every placed module has a grant at the fingerprint the indexer names"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
p.missingSaid, p.missingKnown = why, true
|
||||
}
|
||||
}
|
||||
|
||||
// GrantsOf is the grant of every module at the fingerprint the indexer's trunk names for it, by key, and the keys that
|
||||
// have none (drop): see the comment above. A module in skip (its manifest was refused) is neither. Pure.
|
||||
func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[string]delivery.ControllerGrant, []string, error) {
|
||||
principals, err := buscompose.Users(recs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
accounts, missing := map[string][]string{}, map[string]bool{}
|
||||
for _, p := range principals {
|
||||
if p.Module == "" || (p.Kind != buscompose.KindModule && p.Kind != buscompose.KindProcess) {
|
||||
continue
|
||||
}
|
||||
if in.Hashes[p.Username()].Hash == "" {
|
||||
missing[p.Module] = true
|
||||
continue
|
||||
}
|
||||
accounts[p.Module] = append(accounts[p.Module], p.Username())
|
||||
}
|
||||
out := map[string]delivery.ControllerGrant{}
|
||||
var drop []string
|
||||
for module, fp := range in.Fingerprints {
|
||||
if skip[module] || !strings.HasPrefix(fp, delivery.SourcePrefix2) {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PartGrantKey(module, fp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
names := accounts[module]
|
||||
if missing[module] && len(names) == 0 {
|
||||
drop = append(drop, key)
|
||||
continue
|
||||
}
|
||||
sort.Strings(names)
|
||||
out[key] = delivery.ControllerGrant{Accounts: names}
|
||||
}
|
||||
sort.Strings(drop)
|
||||
return out, drop, nil
|
||||
}
|
||||
|
||||
// grantsReadFailed names a read of this pass without which no grant is written: the hashes and the indexer's modules and
|
||||
// manifests. A manifest refused for one module only leaves that module out (writeGrants).
|
||||
func grantsReadFailed(in Inputs) string {
|
||||
for _, g := range in.Gaps {
|
||||
if !g.Failed {
|
||||
continue
|
||||
}
|
||||
for _, input := range []string{ReadAccounts, ReadModules, ReadManifests} {
|
||||
if strings.HasPrefix(g.Input, input) {
|
||||
return g.Input + ": " + g.Why
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a
|
||||
// module that has none now.
|
||||
func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) {
|
||||
if why := grantsReadFailed(in); why != "" {
|
||||
res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}}
|
||||
return
|
||||
}
|
||||
skip := map[string]bool{}
|
||||
for _, g := range in.Refused {
|
||||
skip[strings.TrimPrefix(g.Input, "manifest of ")] = true
|
||||
}
|
||||
want, drop, err := GrantsOf(in, recs, skip)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, "the grants could not be composed: "+err.Error())
|
||||
return
|
||||
}
|
||||
done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)}
|
||||
res.Grants = done
|
||||
p.sayMissing(done.Missing)
|
||||
for _, key := range sortedKeys(want) {
|
||||
g := want[key]
|
||||
data, err := delivery.EncodeControllerGrant(g)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("the grant %s cannot be encoded: %v", key, err))
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := p.Store.Get(StateGrants, key); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateGrants, key, err))
|
||||
continue
|
||||
} else if found {
|
||||
if was, err := delivery.DecodeControllerGrant(raw); err == nil && delivery.SameControllerGrant(was, withVersion(g)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, err := p.Store.Put(StateGrants, key, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err))
|
||||
continue
|
||||
}
|
||||
done.Written = append(done.Written, key)
|
||||
module, fp, _ := delivery.ParsePartGrantKey(key)
|
||||
p.say(res, EventGrantWritten, map[string]any{"module": module, "fingerprint": fp, "accounts": g.Accounts})
|
||||
}
|
||||
for _, key := range drop {
|
||||
if _, _, found, err := p.Store.Get(StateGrants, key); err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if err := p.Store.Delete(StateGrants, key); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateGrants, key, err))
|
||||
continue
|
||||
}
|
||||
done.Deleted = append(done.Deleted, key)
|
||||
}
|
||||
}
|
||||
|
||||
func withVersion(g delivery.ControllerGrant) delivery.ControllerGrant {
|
||||
g.V = delivery.Version
|
||||
return g
|
||||
}
|
||||
|
||||
// GrantDifference is one key on which the part's grants and the controller's differ.
|
||||
type GrantDifference struct {
|
||||
Key string `json:"key"`
|
||||
// Kind is `only the controller's`, `only the part's` or `accounts differ`.
|
||||
Kind string `json:"kind"`
|
||||
Controller []string `json:"controller_accounts,omitempty"`
|
||||
Part []string `json:"part_accounts,omitempty"`
|
||||
}
|
||||
|
||||
// CompareGrants lists every key the two sets differ on, in key order: a key held by one alone, or held by both with other
|
||||
// accounts. Pure.
|
||||
func CompareGrants(controller, part map[string]delivery.ControllerGrant) []GrantDifference {
|
||||
keys := map[string]bool{}
|
||||
for k := range controller {
|
||||
keys[k] = true
|
||||
}
|
||||
for k := range part {
|
||||
keys[k] = true
|
||||
}
|
||||
out := []GrantDifference{}
|
||||
for _, k := range sortedKeys(keys) {
|
||||
c, inC := controller[k]
|
||||
g, inP := part[k]
|
||||
switch {
|
||||
case inC && !inP:
|
||||
out = append(out, GrantDifference{Key: k, Kind: "only the controller's", Controller: c.Accounts})
|
||||
case inP && !inC:
|
||||
out = append(out, GrantDifference{Key: k, Kind: "only the part's", Part: g.Accounts})
|
||||
case !delivery.SameControllerGrant(withVersion(c), withVersion(g)):
|
||||
out = append(out, GrantDifference{Key: k, Kind: "accounts differ", Controller: c.Accounts, Part: g.Accounts})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -38,9 +38,11 @@ const (
|
||||
ReadPlacement = delivery.PlacementRead
|
||||
)
|
||||
|
||||
// Reads is every state the part reads, in the manifest's order.
|
||||
// Reads is every state the part reads, in the manifest's order. ReadAssignments stays (and in Followed) only while the
|
||||
// controller holds the lease, when the assignments are unioned with the placement (ReadInputs); after the lease write
|
||||
// they place nothing, and the first change after it takes them out (../assignments-readers names the part until then).
|
||||
var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
|
||||
ReadAccounts, ReadPlacement}
|
||||
ReadAccounts, ReadPlacement, ReadLease}
|
||||
|
||||
// The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
|
||||
const (
|
||||
@@ -105,7 +107,7 @@ type resolved struct {
|
||||
type Inputs struct {
|
||||
// Nodes is every machine seen: in a node agent's have or a standing assignment.
|
||||
Nodes []string
|
||||
// Assigned is each machine's standing assignments and what the catalogue's placement puts there, sorted: while both
|
||||
// Assigned is each machine's standing assignments and what the seat's placement puts there, sorted: while both
|
||||
// stand (ADR 0549 §5) a module either names keeps its membership.
|
||||
Assigned map[string][]string
|
||||
// Placed is each node's placement as the seat mesh-placement holds it (ADR 0567 §1); empty before any is read.
|
||||
@@ -115,6 +117,12 @@ type Inputs struct {
|
||||
Manifests map[string]catalogue.Manifest
|
||||
// Digests is each module's manifest digest, by module.
|
||||
Digests map[string]string
|
||||
// Fingerprints is the fingerprint of the new form the indexer's trunk value names for each module, by module: what a
|
||||
// grant is written for (grants.go).
|
||||
Fingerprints map[string]string
|
||||
// NoFingerprint is why the indexer's value of a module names no fingerprint of the new form, by module (its none2, or
|
||||
// that it names none): a module here gets no grant, and that is said (grants.go MissingGrants).
|
||||
NoFingerprint map[string]string
|
||||
// Running is the manifest a machine is wanted to run, by `<node>.<module>`: the one the builder built at the
|
||||
// fingerprint the assigner's want names. An assignment with none is composed from the trunk's manifest.
|
||||
Running map[string]catalogue.Manifest
|
||||
@@ -138,6 +146,22 @@ type HeldHash struct {
|
||||
Generation int `json:"generation,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Adopted bool `json:"adopted,omitempty"`
|
||||
// Fingerprint is the vault's fingerprint of the value the hash is of: with the generation, what says the password
|
||||
// changed, never the hash itself (a hash of the same password differs by its salt).
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// Rotating is the vault's rotation of the account under way: a change of its hash it announces (ADR 0561 §4, §7).
|
||||
Rotating *struct {
|
||||
Login string `json:"login"`
|
||||
} `json:"rotating,omitempty"`
|
||||
}
|
||||
|
||||
// credentialOf is what says which password a held hash is of, never the hash: adopted, or the vault's generation and the
|
||||
// fingerprint of its value.
|
||||
func credentialOf(h HeldHash) string {
|
||||
if h.Adopted {
|
||||
return "adopted"
|
||||
}
|
||||
return fmt.Sprintf("generation %d %s", h.Generation, h.Fingerprint)
|
||||
}
|
||||
|
||||
// readHashes reads the vault's bus-accounts; a refusal is a gap, and every user then has no hash held.
|
||||
@@ -221,19 +245,23 @@ func each(r Reader, state string, gaps *[]Gap) map[string][]byte {
|
||||
|
||||
// ReadInputs reads what the shadow composes from. It never fails: what it cannot read is a gap.
|
||||
func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{},
|
||||
Running: map[string]catalogue.Manifest{},
|
||||
Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
|
||||
in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{}, Fingerprints: map[string]string{},
|
||||
NoFingerprint: map[string]string{}, Running: map[string]catalogue.Manifest{},
|
||||
Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
|
||||
nodes := map[string]bool{}
|
||||
// **After the lease write the assignments place nothing** (novox/hq ADR 0549 §3, §6): they are frozen then, so a
|
||||
// module the catalogue takes off a node would keep its account, grant, membership and user-list entry. Once the lease
|
||||
// is handed away and the placement is read, the placement alone places modules, as the assigner's Placed does with
|
||||
// the same lease reading (buscompose.LeaseHandedAway); before, or with no placement on the bus, the two are unioned.
|
||||
byAssignment, busByAssignment := map[string][]string{}, ""
|
||||
for _, raw := range each(r, ReadAssignments, &in.Gaps) {
|
||||
var a assignment
|
||||
if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" {
|
||||
continue
|
||||
}
|
||||
nodes[a.Node] = true
|
||||
in.Assigned[a.Node] = append(in.Assigned[a.Node], a.Module)
|
||||
byAssignment[a.Node] = append(byAssignment[a.Node], a.Module)
|
||||
if a.Module == BusModule {
|
||||
in.BusNode = a.Node
|
||||
busByAssignment = a.Node
|
||||
}
|
||||
}
|
||||
// The seat's placement (ADR 0567 §9): what it places is composed beside the assignments, and the seats held once
|
||||
@@ -257,11 +285,27 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
in.Placed = map[string]delivery.Placement{}
|
||||
}
|
||||
in.Placed[node] = p
|
||||
}
|
||||
}
|
||||
away, _ := leaseHandedAway(r)
|
||||
if away && len(in.Placed) > 0 {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "not used: the controller's lease is handed away, " +
|
||||
"so the seat's placement alone places modules (ADR 0549 §3, §6)"})
|
||||
} else {
|
||||
for node, ms := range byAssignment {
|
||||
nodes[node] = true
|
||||
for _, m := range p.Assigned() {
|
||||
if !slices.Contains(in.Assigned[node], m) {
|
||||
in.Assigned[node] = append(in.Assigned[node], m)
|
||||
}
|
||||
in.Assigned[node] = append(in.Assigned[node], ms...)
|
||||
}
|
||||
in.BusNode = busByAssignment
|
||||
}
|
||||
for node, p := range in.Placed {
|
||||
nodes[node] = true
|
||||
for _, m := range p.Assigned() {
|
||||
if !slices.Contains(in.Assigned[node], m) {
|
||||
in.Assigned[node] = append(in.Assigned[node], m)
|
||||
}
|
||||
if m == BusModule && in.BusNode == "" {
|
||||
in.BusNode = node
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -322,6 +366,15 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
if err == nil && v.Manifest == "" {
|
||||
err = fmt.Errorf("the indexer's value names no manifest digest")
|
||||
}
|
||||
if err == nil && strings.HasPrefix(v.FP2, delivery.SourcePrefix2) {
|
||||
in.Fingerprints[module] = v.FP2
|
||||
} else if err == nil {
|
||||
why := v.None2
|
||||
if why == "" {
|
||||
why = "the indexer's value names no fingerprint of the new form"
|
||||
}
|
||||
in.NoFingerprint[module] = why
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err == nil {
|
||||
m, err = cache.get(r, v.Manifest)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
@@ -23,31 +24,38 @@ const (
|
||||
StateMemberships = "memberships"
|
||||
StateAssertions = "assertions"
|
||||
StateTakeover = "takeover"
|
||||
KeyList = "list"
|
||||
KeySwitch = "switch"
|
||||
// StateGrants is the part's grants by module and fingerprint, the key layout of the controller's (grants.go).
|
||||
StateGrants = delivery.PartGrantsState
|
||||
KeyList = "list"
|
||||
KeySwitch = "switch"
|
||||
// KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList.
|
||||
KeyPass = "pass"
|
||||
KeyAsked = "asked"
|
||||
)
|
||||
|
||||
// States is every state the part keeps.
|
||||
var States = []string{StateAssertions, StateMemberships, StateTakeover, StateUserList}
|
||||
var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList}
|
||||
|
||||
// The events the part says.
|
||||
const (
|
||||
EventUserList = "user-list-changed"
|
||||
EventMembership = "membership-changed"
|
||||
// EventGrantWritten is said when the part wrote a grant of its state grants (ADR 0547 §5, ADR 0561 §2).
|
||||
EventGrantWritten = "grant-written"
|
||||
// EventAsserted is said when the part created a bucket, log or stream a manifest declares (issue 1790).
|
||||
EventAsserted = "asserted"
|
||||
)
|
||||
|
||||
// Emits is every event the part says.
|
||||
var Emits = []string{EventAsserted, EventMembership, EventUserList}
|
||||
var Emits = []string{EventAsserted, EventGrantWritten, EventHashesAccepted, EventMembership, EventUserList}
|
||||
|
||||
// Store is the part's own state, by local name: the only thing the part writes.
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, uint64, bool, error)
|
||||
Put(state, key string, value []byte) (uint64, error)
|
||||
// Keys and Delete are for the wanted accounts (accounts.go) and the grants (grants.go): every other state is only put.
|
||||
Keys(state string) ([]string, error)
|
||||
Delete(state, key string) error
|
||||
}
|
||||
|
||||
// Emitter says one of the part's events.
|
||||
@@ -111,9 +119,15 @@ type Result struct {
|
||||
// free, <why>`: a verified sender is composed as from no root-free machine there (issue 1945).
|
||||
RootNotFree []string `json:"root_not_free,omitempty"`
|
||||
// Sealed is the list sealed to the bus node as the key `user-list.sealed` names it (ADR 0547 clause 7).
|
||||
Sealed *SealedState `json:"sealed,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
PublishedOrAsserted string `json:"published_or_asserted"`
|
||||
Sealed *SealedState `json:"sealed,omitempty"`
|
||||
// Consumers is what the pass found and created of the durable consumers an account needs (ADR 0561 §6).
|
||||
Consumers *ConsumersState `json:"consumers,omitempty"`
|
||||
// Accounts is what the pass wrote of the wanted accounts the vault mints from (ADR 0561 §2).
|
||||
Accounts *AccountsWritten `json:"accounts,omitempty"`
|
||||
// Grants is what the pass wrote of its grants (grants.go).
|
||||
Grants *GrantsWritten `json:"grants,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
PublishedOrAsserted string `json:"published_or_asserted"`
|
||||
}
|
||||
|
||||
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
|
||||
@@ -136,6 +150,12 @@ type Part struct {
|
||||
Logf func(string, ...any)
|
||||
// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools.
|
||||
Objects Objects
|
||||
// Consumers creates an account's durable consumer from the lease write (consumers.go): the process's connection;
|
||||
// nil in the tools, which compare only.
|
||||
Consumers ConsumerCreator
|
||||
// Bus is where the nodes reach the bus, which each wanted account carries (accounts.go): the process's own credential's;
|
||||
// empty in the tools, which write no account.
|
||||
Bus BusAddress
|
||||
// Say says a condition to the operator: the process's; nil in the tools.
|
||||
Say func(asks.Condition) error
|
||||
// Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix.
|
||||
@@ -157,6 +177,14 @@ type Part struct {
|
||||
holdMu sync.Mutex
|
||||
heldSaid string
|
||||
heldKnown bool
|
||||
// differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go).
|
||||
differSaid string
|
||||
differKnown bool
|
||||
// missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go).
|
||||
missingSaid string
|
||||
missingKnown bool
|
||||
// hashHold are the users whose hash changes held the pass's list, or nil (sealed.go).
|
||||
hashHold []string
|
||||
}
|
||||
|
||||
// Ask is the value of KeyAsked: a tool's ask to compose now, and, from grants_assert, to create one module's missing
|
||||
@@ -165,18 +193,21 @@ type Ask struct {
|
||||
By string `json:"by"`
|
||||
At time.Time `json:"at"`
|
||||
Assert string `json:"assert,omitempty"`
|
||||
// Consumer is, from grants_consumer_match, one account consumer `<stream>.<name>` to bring to the composed
|
||||
// configuration by hand (ADR 0561 §6).
|
||||
Consumer string `json:"consumer,omitempty"`
|
||||
}
|
||||
|
||||
// askedAssert is the module a new ask names to assert by hand, read once per ask; empty when none.
|
||||
func (p *Part) askedAssert() string {
|
||||
// asked is a new ask, read once per ask; the zero Ask when none.
|
||||
func (p *Part) asked() Ask {
|
||||
raw, rev, found, err := p.Store.Get(StateUserList, KeyAsked)
|
||||
if err != nil || !found || rev == p.askSeen {
|
||||
return ""
|
||||
return Ask{}
|
||||
}
|
||||
p.askSeen = rev
|
||||
var a Ask
|
||||
_ = json.Unmarshal(raw, &a)
|
||||
return a.Assert
|
||||
return a
|
||||
}
|
||||
|
||||
func (p *Part) now() time.Time {
|
||||
@@ -240,6 +271,14 @@ func (p *Part) Compose() Result {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The accounts the vault mints from, said in the state only it reads (ADR 0561 §2).
|
||||
if p.Store != nil {
|
||||
p.writeAccounts(&res, in, recs)
|
||||
}
|
||||
// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5).
|
||||
if p.Store != nil {
|
||||
p.writeGrants(&res, in, recs)
|
||||
}
|
||||
// The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7).
|
||||
if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil {
|
||||
res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error())
|
||||
@@ -274,10 +313,11 @@ func (p *Part) Compose() Result {
|
||||
}
|
||||
}
|
||||
held, heldErr := HeldStreams(p.Call)
|
||||
forced := ""
|
||||
if p.Create != nil {
|
||||
forced = p.askedAssert()
|
||||
var ask Ask
|
||||
if p.Create != nil || p.Consumers != nil {
|
||||
ask = p.asked()
|
||||
}
|
||||
forced := ask.Assert
|
||||
for _, module := range sortedKeys(in.Manifests) {
|
||||
if !assigned(in, module) {
|
||||
continue
|
||||
@@ -299,6 +339,7 @@ func (p *Part) Compose() Result {
|
||||
res.AssertionsWritten++
|
||||
}
|
||||
}
|
||||
p.consumers(&res, recs, ask.Consumer)
|
||||
if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found {
|
||||
data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}})
|
||||
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
|
||||
@@ -324,6 +365,11 @@ func (p *Part) Last() Result {
|
||||
}
|
||||
|
||||
func assigned(in Inputs, module string) bool {
|
||||
// The node-engine runs on every node though no node is assigned it (novox/hq ADR 0563 §2, §3): its manifest is
|
||||
// placed everywhere, so its declared state is asserted and created as a placed module's.
|
||||
if module == buscompose.EngineModule {
|
||||
return true
|
||||
}
|
||||
for _, ms := range in.Assigned {
|
||||
for _, m := range ms {
|
||||
if m == module {
|
||||
|
||||
@@ -78,6 +78,11 @@ func Records(in Inputs) (buscompose.Records, []Gap) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The node-engine's own manifest, placed on every node (ADR 0563): its state, logs and streams alone, created by this
|
||||
// part; it adds no principal, since each node's engine speaks as the node's account.
|
||||
if m, ok := in.Manifests[buscompose.EngineModule]; ok {
|
||||
out.Engine = &buscompose.Declared{Module: m.Module, State: bucketsOf(m), Logs: logsOf(m), Streams: streamsOf(m)}
|
||||
}
|
||||
return out, gaps
|
||||
}
|
||||
|
||||
|
||||
@@ -23,8 +23,17 @@ const (
|
||||
KindRootNotFree = "root-not-free"
|
||||
)
|
||||
|
||||
// EventAccountIssued is the vault's word that it minted an account (novox/hq ADR 0561 §1): its hash is in the vault's
|
||||
// bus-accounts, so a pass composes the list with it.
|
||||
const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent
|
||||
|
||||
// Consumes is every event the part hears, as its manifest's consumes name them.
|
||||
var Consumes = []string{EventConditionRaised, EventConditionCleared}
|
||||
var Consumes = []string{EventConditionRaised, EventConditionCleared, EventAccountIssued}
|
||||
|
||||
// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject.
|
||||
func IsAccountIssued(event string) bool {
|
||||
return event == EventAccountIssued || event == buscompose.AccountIssuedSubject
|
||||
}
|
||||
|
||||
// EventSubject is the bus subject of a controller seat event named `mesh-controller.<event>`.
|
||||
func EventSubject(event string) string {
|
||||
|
||||
@@ -2,10 +2,12 @@ package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
@@ -91,11 +93,12 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string
|
||||
why = "no node runs the bus's module, so the list is sealed to nobody"
|
||||
case key == "":
|
||||
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
|
||||
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost) != "":
|
||||
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "":
|
||||
// After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user
|
||||
// of it for want of a hash: that user's module would be locked out of the bus (review of the switch).
|
||||
why = p.holdsBack(in, leftOut, bus, lost)
|
||||
why = p.holdsBack(in, leftOut, bus, lost, digest)
|
||||
held = why
|
||||
p.keepHashHold(res, digest)
|
||||
case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode:
|
||||
object, err := delivery.SealUserList(list, key)
|
||||
if err == nil {
|
||||
@@ -133,7 +136,8 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string
|
||||
return
|
||||
}
|
||||
if next.Digest == digest && cur.Digest != digest {
|
||||
p.keepWrittenUsers(res, list)
|
||||
p.keepWrittenUsers(res, list, in.Hashes)
|
||||
p.acceptedWritten(res, digest)
|
||||
}
|
||||
// The object the key named before is taken away once the key names the new one: the bucket keeps the list in force.
|
||||
if had && cur.Object != "" && cur.Object != next.Object {
|
||||
@@ -288,11 +292,50 @@ func (p *Part) followReasons(res Result, in Inputs) []string {
|
||||
// composed after take-over must keep.
|
||||
const KeyWrittenUsers = "sealed-users"
|
||||
|
||||
func (p *Part) keepWrittenUsers(res *Result, list []byte) {
|
||||
data, _ := json.Marshal(listUsers(list))
|
||||
// KeyWrittenCredentials is the key of the state user-list naming, per user of the list in force, which password its hash
|
||||
// is of (credentialOf: adopted, or the vault's generation and fingerprint), never the hash: what tells a list that changes
|
||||
// several loaded users' hashes at once (ADR 0561 §7).
|
||||
const KeyWrittenCredentials = "sealed-credentials"
|
||||
|
||||
func (p *Part) keepWrittenUsers(res *Result, list []byte, hashes map[string]HeldHash) {
|
||||
users := listUsers(list)
|
||||
data, _ := json.Marshal(users)
|
||||
if _, err := p.Store.Put(StateUserList, KeyWrittenUsers, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyWrittenUsers, err))
|
||||
}
|
||||
creds := map[string]string{}
|
||||
for _, u := range users {
|
||||
if h, ok := hashes[u]; ok {
|
||||
creds[u] = credentialOf(h)
|
||||
}
|
||||
}
|
||||
data, _ = json.Marshal(creds)
|
||||
if _, err := p.Store.Put(StateUserList, KeyWrittenCredentials, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyWrittenCredentials, err))
|
||||
}
|
||||
}
|
||||
|
||||
// changedHashes are the users of the list in force whose hash the list composed now would change with no rotation by the
|
||||
// vault naming them, sorted; none when the part kept no credentials of the list in force (a list written before).
|
||||
func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) {
|
||||
raw, _, found, err := p.Store.Get(StateUserList, KeyWrittenCredentials)
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
var had map[string]string
|
||||
if err := json.Unmarshal(raw, &had); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var changed []string
|
||||
for user, was := range had {
|
||||
h, ok := hashes[user]
|
||||
if !ok || credentialOf(h) == was || h.Rotating != nil {
|
||||
continue
|
||||
}
|
||||
changed = append(changed, user)
|
||||
}
|
||||
slices.Sort(changed)
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
var listUser = regexp.MustCompile(`\{\s*user:\s*"([^"]+)"`)
|
||||
@@ -312,7 +355,8 @@ func listUsers(list []byte) []string {
|
||||
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
|
||||
// it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
|
||||
// they could not be read). Empty when it may.
|
||||
func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string) string {
|
||||
func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string {
|
||||
p.hashHold = nil
|
||||
if f := in.ReadFailures(); len(f) > 0 {
|
||||
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
|
||||
}
|
||||
@@ -342,5 +386,127 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string) string {
|
||||
if len(unhashed) > 0 {
|
||||
return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault"
|
||||
}
|
||||
// More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new
|
||||
// writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21.
|
||||
changed, err := p.changedHashes(in.Hashes)
|
||||
if err != nil {
|
||||
return "held: the credentials of the list in force cannot be read: " + err.Error()
|
||||
}
|
||||
if len(changed) > 1 && !p.accepted(digest, changed) {
|
||||
p.hashHold = changed
|
||||
return "held: the list in force is kept, since the hashes of " + fmt.Sprint(len(changed)) + " loaded users would change at " +
|
||||
"once with no rotation by the vault naming them: " + strings.Join(changed, ", ")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// **A person accepts a held list's hash changes** (ADR 0561 §7, §9; tools-are-how-a-module-is-spoken-to): the hold of
|
||||
// several loaded users' hashes changing at once is released only by grants_accept_hashes, naming exactly the users held,
|
||||
// with why, bound to the held list's digest. The acceptance releases this guard alone, once: the other holds (a failed
|
||||
// read, a user dropped for want of a hash, a lost right) stand whatever it says; and a list composed otherwise since is
|
||||
// held again, for a new acceptance.
|
||||
|
||||
// KeyHashHold names the list held for its hash changes (its digest and the users), and KeyHashAccept a person's acceptance.
|
||||
const (
|
||||
KeyHashHold = "hash-hold"
|
||||
KeyHashAccept = "hash-accept"
|
||||
)
|
||||
|
||||
// EventHashesAccepted is said when a list held for its hash changes is written on a person's acceptance: a hand act.
|
||||
const EventHashesAccepted = "hashes-accepted"
|
||||
|
||||
// HashHold is the value of KeyHashHold, and HashAccept of KeyHashAccept.
|
||||
type HashHold struct {
|
||||
Digest string `json:"digest"`
|
||||
Users []string `json:"users"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type HashAccept struct {
|
||||
Digest string `json:"digest"`
|
||||
Users []string `json:"users"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// keepHashHold writes which list is held for its hash changes, when that is the hold.
|
||||
func (p *Part) keepHashHold(res *Result, digest string) {
|
||||
if p.hashHold == nil {
|
||||
return
|
||||
}
|
||||
data, _ := json.Marshal(HashHold{Digest: digest, Users: p.hashHold, At: p.now()})
|
||||
if raw, _, found, err := p.Store.Get(StateUserList, KeyHashHold); err == nil && found {
|
||||
var was HashHold
|
||||
if json.Unmarshal(raw, &was) == nil && was.Digest == digest && slices.Equal(was.Users, p.hashHold) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := p.Store.Put(StateUserList, KeyHashHold, data); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyHashHold, err))
|
||||
}
|
||||
}
|
||||
|
||||
// accepted says a person accepted exactly these users' hash changes of the list of this digest.
|
||||
func (p *Part) accepted(digest string, users []string) bool {
|
||||
raw, _, found, err := p.Store.Get(StateUserList, KeyHashAccept)
|
||||
if err != nil || !found {
|
||||
return false
|
||||
}
|
||||
var a HashAccept
|
||||
return json.Unmarshal(raw, &a) == nil && a.Digest == digest && slices.Equal(a.Users, users)
|
||||
}
|
||||
|
||||
// acceptedWritten says, once, that the list a person accepted is written, and forgets the acceptance and the hold.
|
||||
func (p *Part) acceptedWritten(res *Result, digest string) {
|
||||
raw, _, found, err := p.Store.Get(StateUserList, KeyHashAccept)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
var a HashAccept
|
||||
if json.Unmarshal(raw, &a) != nil || a.Digest != digest {
|
||||
return
|
||||
}
|
||||
p.say(res, EventHashesAccepted, map[string]any{"digest": digest, "users": a.Users, "by": a.By, "why": a.Why})
|
||||
for _, k := range []string{KeyHashAccept, KeyHashHold} {
|
||||
if err := p.Store.Delete(StateUserList, k); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, k, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AcceptHashes is the tool accept-hashes: a person accepts the hash changes of the list held for them, naming exactly its
|
||||
// users, with why. Refused when no list is held for its hashes, or for another set of users. Bound to the held list's
|
||||
// digest: a list composed otherwise since is held again.
|
||||
func (p *Part) AcceptHashes(caller string, users []string, why string) (any, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("accept-hashes says why (`why`)")
|
||||
}
|
||||
raw, _, found, err := p.Store.Get(StateUserList, KeyHashHold)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the held list cannot be read: %w", err)
|
||||
}
|
||||
var h HashHold
|
||||
if !found || json.Unmarshal(raw, &h) != nil || h.Digest == "" {
|
||||
return nil, errors.New("no list is held for its users' hashes changing: there is nothing to accept")
|
||||
}
|
||||
named := slices.Sorted(slices.Values(users))
|
||||
if !slices.Equal(named, h.Users) {
|
||||
return nil, fmt.Errorf("the list held (%s) changes the hashes of %s: an acceptance names exactly those, and %s is not that",
|
||||
short(h.Digest), strings.Join(h.Users, ", "), strings.Join(named, ", "))
|
||||
}
|
||||
data, _ := json.Marshal(HashAccept{Digest: h.Digest, Users: h.Users, By: someoneOr(caller), Why: strings.TrimSpace(why), At: p.now()})
|
||||
rev, err := p.Store.Put(StateUserList, KeyHashAccept, data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the acceptance could not be written: %w", err)
|
||||
}
|
||||
return map[string]any{"accepted": rev, "digest": h.Digest, "users": h.Users, "says": "the part's process writes the held " +
|
||||
"list at its next pass, once, unless another hold stands or the list composed then is another; it says hashes-accepted"}, nil
|
||||
}
|
||||
|
||||
func someoneOr(caller string) string {
|
||||
if caller == "" {
|
||||
return "a caller the runtime did not prove"
|
||||
}
|
||||
return caller
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -21,43 +20,92 @@ const (
|
||||
ToolTakeOver = "grants_take_over"
|
||||
ToolTakeOverState = "grants_take_over_state"
|
||||
ToolHandBack = "grants_hand_back"
|
||||
ToolAccounts = "grants_accounts"
|
||||
ToolConsumerMatch = "grants_consumer_match"
|
||||
ToolAcceptHashes = "grants_accept_hashes"
|
||||
)
|
||||
|
||||
// Tools is every tool, in the manifest's order.
|
||||
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState}
|
||||
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState,
|
||||
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes}
|
||||
|
||||
// WritesNoGrant is what grants says of the part's own grants in this milestone.
|
||||
const WritesNoGrant = "none: the grants part writes no grants.<module>.<fp> before ADR 0547's third milestone (clause 5); the controller's are beside"
|
||||
// WritesGrants is what grants says of the part's own grants in this milestone: written, and read by nobody until the
|
||||
// controller's lease is handed away (delivery.GrantSource).
|
||||
const WritesGrants = "in shadow: the grants part writes grants.<module>.<fp> into its state grants, which the assigner reads only once the controller's lease is handed away (ADR 0547 clause 5); the controller's are beside, with every difference"
|
||||
|
||||
// Grants answers the tool grants: what the part writes of grants (none), and the controller's, one module's or all.
|
||||
// Grants answers the tool grants: the grants the part wrote, the controller's, and every difference between them, one
|
||||
// module's or all.
|
||||
func (p *Part) Grants(module string) (any, error) {
|
||||
keys, err := p.Read.Keys(ReadControllerGrant)
|
||||
if err != nil {
|
||||
return map[string]any{"writes": WritesNoGrant, "controller_unread": err.Error()}, nil
|
||||
}
|
||||
type grant struct {
|
||||
Module string `json:"module"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Accounts []string `json:"accounts,omitempty"`
|
||||
}
|
||||
out := []grant{}
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseControllerGrantKey(k)
|
||||
if err != nil || (module != "" && m != module) {
|
||||
continue
|
||||
out := map[string]any{"writes": WritesGrants}
|
||||
read := func(get func() ([]string, func(string) ([]byte, bool, error), error)) (map[string]delivery.ControllerGrant, error) {
|
||||
keys, one, err := get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g := grant{Module: m, Fingerprint: fp}
|
||||
if module != "" {
|
||||
if raw, found, err := p.Read.Get(ReadControllerGrant, k); err == nil && found {
|
||||
if v, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
g.Accounts = v.Accounts
|
||||
}
|
||||
m := map[string]delivery.ControllerGrant{}
|
||||
for _, k := range keys {
|
||||
mod, _, err := delivery.ParseControllerGrantKey(k)
|
||||
if err != nil || (module != "" && mod != module) {
|
||||
continue
|
||||
}
|
||||
raw, found, err := one(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
g, err := delivery.DecodeControllerGrant(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", k, err)
|
||||
}
|
||||
m[k] = g
|
||||
}
|
||||
out = append(out, g)
|
||||
return m, nil
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Fingerprint < out[j].Module+out[j].Fingerprint })
|
||||
return map[string]any{"writes": WritesNoGrant, "controller": out, "count": len(out)}, nil
|
||||
controller, cerr := read(func() ([]string, func(string) ([]byte, bool, error), error) {
|
||||
keys, err := p.Read.Keys(ReadControllerGrant)
|
||||
return keys, func(k string) ([]byte, bool, error) { return p.Read.Get(ReadControllerGrant, k) }, err
|
||||
})
|
||||
mine, perr := read(func() ([]string, func(string) ([]byte, bool, error), error) {
|
||||
keys, err := p.Store.Keys(StateGrants)
|
||||
return keys, func(k string) ([]byte, bool, error) {
|
||||
raw, _, found, err := p.Store.Get(StateGrants, k)
|
||||
return raw, found, err
|
||||
}, err
|
||||
})
|
||||
type grant struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
Accounts []string `json:"accounts,omitempty"`
|
||||
}
|
||||
list := func(m map[string]delivery.ControllerGrant) []grant {
|
||||
out := []grant{}
|
||||
for _, k := range sortedKeys(m) {
|
||||
mod, _, _ := delivery.ParseControllerGrantKey(k)
|
||||
out = append(out, grant{Key: k, Module: mod, Accounts: m[k].Accounts})
|
||||
}
|
||||
return out
|
||||
}
|
||||
if cerr != nil {
|
||||
out["controller_unread"] = cerr.Error()
|
||||
} else {
|
||||
out["controller"], out["controller_count"] = list(controller), len(controller)
|
||||
}
|
||||
if perr != nil {
|
||||
out["written_unread"] = perr.Error()
|
||||
} else {
|
||||
out["written"], out["count"] = list(mine), len(mine)
|
||||
}
|
||||
if cerr == nil && perr == nil {
|
||||
diff := CompareGrants(controller, mine)
|
||||
out["differences"], out["differing"] = diff, len(diff)
|
||||
out["equal"] = len(diff) == 0
|
||||
}
|
||||
if last := p.Last(); last.Grants != nil {
|
||||
out["last_pass"] = last.Grants
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MembershipOf answers the tool membership: one module on one node as composed now, and as last published by the
|
||||
|
||||
@@ -12,11 +12,13 @@
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"node-engine.have",
|
||||
"mesh-vault.bus-accounts"
|
||||
"mesh-vault.bus-accounts",
|
||||
"mesh-controller.lease"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-controller.condition-raised",
|
||||
"mesh-controller.condition-cleared"
|
||||
"mesh-controller.condition-cleared",
|
||||
"mesh-vault.account-issued"
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
@@ -29,6 +31,8 @@
|
||||
],
|
||||
"emits": [
|
||||
"asserted",
|
||||
"grant-written",
|
||||
"hashes-accepted",
|
||||
"membership-changed",
|
||||
"user-list-changed"
|
||||
],
|
||||
@@ -40,13 +44,24 @@
|
||||
"grants_assert",
|
||||
"grants_take_over",
|
||||
"grants_take_over_state",
|
||||
"grants_hand_back"
|
||||
"grants_hand_back",
|
||||
"grants_accounts",
|
||||
"grants_consumer_match",
|
||||
"grants_accept_hashes"
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
"name": "accounts",
|
||||
"history": 5
|
||||
},
|
||||
{
|
||||
"name": "assertions",
|
||||
"history": 5
|
||||
},
|
||||
{
|
||||
"name": "grants",
|
||||
"history": 5
|
||||
},
|
||||
{
|
||||
"name": "memberships",
|
||||
"history": 5
|
||||
|
||||
Reference in New Issue
Block a user