From a092ee9ff9df2f496ee3fb7251ad5120f5b6e69d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 11:59:41 +0100 Subject: [PATCH] mesh-grants writes the user list sealed to the bus node and names it in user-list.sealed; the switch turns that key; take-over state says switched only once the bus node loaded it (ADR 0547 clauses 7, 8; issue 1945) The bus node can only take over a list that exists: each pass composes the accounts block with the vault's hashes, seals it to the key the bus node says in its have, puts it in the part's own object bucket by digest and writes the key with digest, revision, user count, users left out and which list to follow. take-over is refused, naming why, while no list is sealed to the bus node's current key or the loaded list does not grant the bus node its read. Review of fix/1945-takeover-blockers: no responders on the router's record is an error, not 'no record'; an answer whose record cannot be reached is not closed; an ask that failed to publish is taken back at the router; an error acting on an answer is logged. Pins mesh-sdk 175a5567. --- modules/mesh-grants/cmd/mesh-grants/asking.go | 5 +- modules/mesh-grants/cmd/mesh-grants/main.go | 10 +- modules/mesh-grants/cmd/mesh-grants/sealed.go | 55 +++++ modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 2 + modules/mesh-grants/grants_test.go | 113 ++++++++- modules/mesh-grants/internal/grants/part.go | 18 +- modules/mesh-grants/internal/grants/sealed.go | 231 ++++++++++++++++++ .../mesh-grants/internal/grants/takeover.go | 48 +++- 9 files changed, 463 insertions(+), 21 deletions(-) create mode 100644 modules/mesh-grants/cmd/mesh-grants/sealed.go create mode 100644 modules/mesh-grants/internal/grants/sealed.go diff --git a/modules/mesh-grants/cmd/mesh-grants/asking.go b/modules/mesh-grants/cmd/mesh-grants/asking.go index 15db062..3095d96 100644 --- a/modules/mesh-grants/cmd/mesh-grants/asking.go +++ b/modules/mesh-grants/cmd/mesh-grants/asking.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "time" "github.com/nats-io/nats.go" @@ -40,8 +41,10 @@ func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id stri ctx, cancel := context.WithTimeout(ctx, recordWithin) defer cancel() msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_asks.$KV.asks."+module+"."+id, nil) + // No responders is the bucket out of reach for a moment, never "no record": read as none, a live ask would be + // closed by Reconcile a minute later and the operator's yes lost. if errors.Is(err, nats.ErrNoResponders) { - return nil, nil + return nil, fmt.Errorf("the router's record of %s cannot be reached: %w", id, err) } if err != nil { return nil, err diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index 365de24..d971edc 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -7,8 +7,9 @@ // module placed since cannot serve its seat or reach a new state. Each is published on the assignments stream only when // it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is // created** (the operator, 2026-10-26 08:54; issue 1790), once per manifest, with the manifest's limits, on the event -// that brought the manifest; one the bus holds is never changed. **The user list, the rest of the assertions and the -// grants stay in shadow** until +// that brought the manifest; one the bus holds is never changed. **The user list is written sealed to the bus node** +// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows only +// from take-over. **The rest of the assertions and the grants stay in shadow** until // the operator's take-over, asked on their channel and run on their yes after its refusals: they are composed, compared with what the bus loaded // and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect. // @@ -116,7 +117,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } bus := newBusReader(ctx, js) f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn}, - Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}} + Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, + Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf} f.Watch = bus // The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on // the warrant once (mesh-sdk asker), after its refusals. @@ -149,7 +151,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) return err } - logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; the user list, grants and any change to a held stream in shadow", account.User) + logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; and writing the user list sealed to the bus node, which the bus node follows from take-over; grants and any change to a held stream in shadow", account.User) f.Run(ctx) return nil } diff --git a/modules/mesh-grants/cmd/mesh-grants/sealed.go b/modules/mesh-grants/cmd/mesh-grants/sealed.go new file mode 100644 index 0000000..1c8156e --- /dev/null +++ b/modules/mesh-grants/cmd/mesh-grants/sealed.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// sealedObjects is the part's object bucket of user lists sealed to the bus node (novox/hq ADR 0547 clause 7), created on +// its first put: the composer grants the part its create, its puts and deletes, and nothing of another bucket. +type sealedObjects struct { + ctx context.Context + js jetstream.JetStream +} + +func (o sealedObjects) store(ctx context.Context) (jetstream.ObjectStore, error) { + s, err := o.js.ObjectStore(ctx, delivery.GrantsUserListObjects) + if errors.Is(err, jetstream.ErrBucketNotFound) { + s, err = o.js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: delivery.GrantsUserListObjects, + Description: "the bus's user list composed by the grants part, each sealed to the bus node's key under the hex of its digest (novox/hq ADR 0547 clause 7): written by mesh-grants alone, read by name by the node-engine of the node holding mesh-broker", + Storage: jetstream.FileStorage, MaxBytes: 64 << 20}) + } + if err != nil { + return nil, fmt.Errorf("the object bucket %s: %w", delivery.GrantsUserListObjects, err) + } + return s, nil +} + +func (o sealedObjects) PutObject(name string, data []byte) error { + ctx, cancel := context.WithTimeout(o.ctx, requestWithin) + defer cancel() + s, err := o.store(ctx) + if err != nil { + return err + } + _, err = s.PutBytes(ctx, name, data) + return err +} + +func (o sealedObjects) DeleteObject(name string) error { + ctx, cancel := context.WithTimeout(o.ctx, requestWithin) + defer cancel() + s, err := o.store(ctx) + if err != nil { + return err + } + if err := s.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 393b876..6ef66ab 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 919fd4e..9c3f761 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896 h1:5rHfru5NJ7SK20QOt8YUJSglcn1gsjtLc9AGmWYGnRQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd h1:ef6y09csev6xY/suFrIFGUXbgvDA29kummFtHcIb3B8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 2d7a5a2..e2f8a60 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "mesh-grants/internal/grants" ) @@ -344,6 +345,7 @@ func TestGrants(t *testing.T) { createOnly := filepath.Join("cmd", "mesh-grants", "create.go") // And the operator's asks and conditions, on the seat operator-channel's subjects, in asking.go (ADR 0547 clause 8). asksOnly := filepath.Join("cmd", "mesh-grants", "asking.go") + sealedOnly := filepath.Join("cmd", "mesh-grants", "sealed.go") files := 0 for _, root := range []string{"cmd", "internal"} { err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { @@ -364,6 +366,13 @@ func TestGrants(t *testing.T) { if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) { t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n) } + // The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7). + if path == sealedOnly { + if strings.Count(string(src), "CreateObjectStore(") != 1 || !strings.Contains(string(src), "delivery.GrantsUserListObjects") { + t.Errorf("sealed.go creates other than the part's one object bucket") + } + src = []byte(strings.ReplaceAll(string(src), "CreateObjectStore", "")) + } for _, f := range forbidden { if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the part holds no timer and creates or asserts nothing", path, f) @@ -730,6 +739,30 @@ func TestGrants(t *testing.T) { b.loaded = shadowOf(t, b) b.streams = []map[string]any{{"name": "KV_mod-x_kept", "subjects": []string{"$KV.mod-x_kept.>"}, "retention": "limits", "max_per_subject": 3}} p := partOn(b) + // The bus node says its key in its have of the bus module; the process puts the sealed list (ADR 0547 clause 7). + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + withKey := func(w delivery.UserListWritten) { + w.Key = busKey.PublicKey + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, UserList: &w}) + } + loadedBefore := delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 6} + withKey(loadedBefore) + objs := &fakeObjects{} + p.Objects = objs + open := func(g delivery.GrantsUserList) string { + t.Helper() + list, err := delivery.OpenUserList(objs.objects[g.Object], g.Digest, func(bx seal.Box) ([]byte, error) { + out, err := seal.Open(bx, busKey.PrivateKey) + return []byte(out), err + }) + if err != nil { + t.Fatalf("the sealed list does not open with the bus node's key: %v", err) + } + return string(list) + } got, _ := p.UserList() if !strings.Contains(fmt.Sprint(got), "nothing composed yet") { @@ -739,6 +772,20 @@ func TestGrants(t *testing.T) { if !first.Written || first.Revision != 1 || !first.UserList.Compare.Equal() || first.UserList.Users != 6 { t.Fatalf("first composition = %+v", first) } + // Written in shadow, the bus node told to follow the controller's list; every user left out while the vault holds + // no hash; a tool's part (no object bucket) writes none. + if g, _, had := p.Sealed(); !had || g.Follow != delivery.FollowController || g.Revision != 1 || g.Node != "a" || + g.Users != 0 || len(g.LeftOut) != 6 || len(objs.objects) != 1 || !strings.Contains(open(g), "accounts {") { + t.Errorf("the sealed list in shadow = %+v, %d objects", g, len(objs.objects)) + } + if _, _, had := partOn(fixture()).Sealed(); had { + t.Error("a part read before any pass holds a sealed list") + } + if tp := partOn(fixture()); tp.Compose().Sealed == nil { + t.Error("a tool's pass does not say the sealed list") + } else if _, _, had := tp.Sealed(); had { + t.Error("a tool's part wrote the sealed list") + } if len(first.Published) != 3 || len(b.publishes) != 3 { t.Errorf("the first composition published %v, want the three assignments' memberships", first.Published) } @@ -889,6 +936,26 @@ func TestGrants(t *testing.T) { if row.outcome == "recorded" && !strings.Contains(fmt.Sprint(p.TakeOverState()), "bus_node_loaded_grants_list:not yet") { t.Errorf("the state claims the bus node follows with no loaded list: %v", fmt.Sprint(p.TakeOverState())) } + // The key the bus node follows turns with the switch at once, and the state says switched only once the bus + // node loaded the grants part's written list. + g, _, _ := p.Sealed() + if want := map[string]string{grants.TookOver: delivery.FollowGrants, grants.Shadow: delivery.FollowController}[p.SwitchState()]; g.Follow != want { + t.Errorf("the key says follow %s with the switch %s", g.Follow, p.SwitchState()) + } + effect := func() string { st, _ := p.TakeOverState(); return fmt.Sprint(st.(map[string]any)["effect"]) } + if row.outcome == "recorded" && row.act == "take-over" { + if e := effect(); strings.HasPrefix(e, "switched") || !strings.Contains(e, "not in effect") { + t.Errorf("effect before the bus node loaded = %s", e) + } + if users := strings.Count(open(g), "{ user: "); users != g.Users || g.Users == 0 || len(g.LeftOut) != 0 { + t.Errorf("the sealed list holds %d users, the key says %d, left out %v", users, g.Users, g.LeftOut) + } + withKey(delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Wanted: g.Digest, Digest: g.Digest, Users: g.Users, Follows: delivery.FollowGrants}) + if e := effect(); !strings.HasPrefix(e, "switched") { + t.Errorf("effect once the bus node loaded the written list = %s", e) + } + withKey(loadedBefore) + } }) } t.Run("hand-back in shadow is refused at once, with nothing asked", func(t *testing.T) { @@ -943,10 +1010,25 @@ func TestGrants(t *testing.T) { id = asked("take-over") p.Compose() p.Compose() - if l := last(); l.Ask != id || l.Said != "refused" || !slices.Contains(forgot, id) { - t.Errorf("a failed ask stays open: %+v, forgot %v", l, forgot) + if l := last(); l.Ask != id || l.Said != "refused" || !slices.Contains(forgot, id) || !slices.Contains(fa.cancels, id) { + t.Errorf("a failed ask stays open, or is not taken back at the router: %+v, forgot %v, cancelled %v", l, forgot, fa.cancels) } fa.fail = false + // An answer the part's own record cannot take yet (its state out of reach) leaves the ask open, closed by + // nothing, and is taken at a later pass. + id = asked("take-over") + p.Compose() + fa.wheres[id] = &asks.Warrant{Ask: id, Outcome: asks.OutcomeChosen, Option: "no"} + fa.takeErr = errors.New("nats: no responders available for request") + p.Compose() + if l := last(); l.Ask != id || l.Said != "asked" { + t.Errorf("an answer not taken closed the ask: %+v", l) + } + fa.takeErr = nil + p.Compose() + if l := last(); l.Ask != id || l.Said != "declined" { + t.Errorf("the answer was not taken once its record could be reached: %+v", l) + } asked("take-over") }) @@ -1259,11 +1341,29 @@ func TestGrants(t *testing.T) { } type fakeAsker struct { - asks []asks.Ask - wheres map[string]*asks.Warrant // the router's warrants, by ask - fail bool + asks []asks.Ask + wheres map[string]*asks.Warrant // the router's warrants, by ask + fail bool + cancels []string + // takeErr makes Take fail as the asker does when the part's own record cannot be reached: no handling, an error. + takeErr error } +func (f *fakeAsker) Cancel(id string) error { f.cancels = append(f.cancels, id); return nil } + +// fakeObjects is the part's object bucket of sealed lists. +type fakeObjects struct{ objects map[string][]byte } + +func (o *fakeObjects) PutObject(name string, data []byte) error { + if o.objects == nil { + o.objects = map[string][]byte{} + } + o.objects[name] = data + return nil +} + +func (o *fakeObjects) DeleteObject(name string) error { delete(o.objects, name); return nil } + func (f *fakeAsker) Ask(a asks.Ask) error { if f.fail { return errors.New("the bus refused the publish") @@ -1276,6 +1376,9 @@ func (f *fakeAsker) Where(id string) (string, *asks.Warrant, error) { return "op // Take hands the warrant to act as the SDK's asker does for one that chose, and records any other. func (f *fakeAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) { + if f.takeErr != nil { + return "", f.takeErr + } if w.Outcome != asks.OutcomeChosen { return asker.Recorded, nil } diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index f864fcc..0c074e0 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -101,9 +101,11 @@ type Result struct { Refused []string `json:"refused,omitempty"` // RootNotFree is each machine the composition needed a root-free word for and did not get one, `: not // free, `: a verified sender is composed as from no root-free machine there (issue 1945). - RootNotFree []string `json:"root_not_free,omitempty"` - Errors []string `json:"errors,omitempty"` - PublishedOrAsserted string `json:"published_or_asserted"` + RootNotFree []string `json:"root_not_free,omitempty"` + // Sealed is the list sealed to the bus node as the key `user-list.sealed` names it (ADR 0547 clause 7). + Sealed *SealedState `json:"sealed,omitempty"` + Errors []string `json:"errors,omitempty"` + PublishedOrAsserted string `json:"published_or_asserted"` } // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. @@ -122,6 +124,10 @@ type Part struct { // Asker asks the operator (take-over, hand-back): the process's client of the router; nil in the tools, which write // the request and let the process ask it. Asker Asker + // Logf logs what no state or condition says (an error of acting on an answer): the process's; nil in the tools. + Logf func(string, ...any) + // Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools. + Objects Objects // Say says a condition to the operator: the process's; nil in the tools. Say func(asks.Condition) error // Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix. @@ -219,6 +225,12 @@ func (p *Part) Compose() Result { } } } + // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). + if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil { + res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error()) + } else { + p.writeSealed(&res, list, n, leftOut, in) + } ms := Memberships(recs) res.Memberships = len(ms) for _, key := range sortedKeys(ms) { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go new file mode 100644 index 0000000..82c047a --- /dev/null +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -0,0 +1,231 @@ +package grants + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **The list the bus node loads from take-over** (novox/hq ADR 0547 clause 7, issue 1945). Each pass composes the bus's +// user list with the vault's hashes (the accounts block, buscompose.ComposeAccounts, as the bus module's file holds it), +// seals it to the bus node's key (the key the node says in its have of the bus module) and puts it as one object of the +// part's object bucket, named by its digest; then writes the key `user-list.sealed` naming it, with the revision, the user +// count, the users left out for want of a hash, and which list the bus node follows (the switch, clause 8). The object +// and the key are written in shadow too: the bus node reads the key and follows the controller's list until the key says +// FollowGrants, so a take-over has a list to load at once. Only the process writes them (Objects); a tool asks it. + +// KeySealed is the key of the state user-list the bus node reads (delivery.GrantsUserListKey). +const KeySealed = delivery.GrantsUserListKey + +// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools. +type Objects interface { + PutObject(name string, data []byte) error + DeleteObject(name string) error +} + +// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault, and the users left out +// for want of one. Pure. +func ComposeList(r buscompose.Records, hashes map[string]HeldHash) ([]byte, int, []string, error) { + principals, err := buscompose.Users(r) + if err != nil { + return nil, 0, nil, err + } + held := map[string]string{} + for user, h := range hashes { + held[user] = h.Hash + } + filled, missing := buscompose.WithPasswords(principals, held) + list, err := buscompose.ComposeAccounts(filled) + if err != nil { + return nil, 0, nil, err + } + slices.Sort(missing) + return []byte(list), len(filled), missing, nil +} + +// Sealed is the part's key `user-list.sealed` as written, and whether there is one. +func (p *Part) Sealed() (delivery.GrantsUserList, uint64, bool) { + raw, rev, found, err := p.Store.Get(StateUserList, KeySealed) + if err != nil || !found { + return delivery.GrantsUserList{}, 0, false + } + g, err := delivery.DecodeGrantsUserList(raw) + if err != nil { + return delivery.GrantsUserList{}, 0, false + } + return g, rev, true +} + +// followOf is which list the switch says the bus node follows. +func followOf(state string) string { + if state == TookOver { + return delivery.FollowGrants + } + return delivery.FollowController +} + +// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in +// res what it wrote, or why it wrote nothing. +func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs) { + cur, _, had := p.Sealed() + next := cur + next.Follow = followOf(p.SwitchState()) + key := "" + if in.Loaded != nil { + key = in.Loaded.Key + } + digest := delivery.UserListDigest(list) + why := "" + switch { + case p.Objects == nil: + // The tools: the process writes; the follow alone may change here, from a tool's view of the switch, and is not. + res.Sealed = sealedSays(cur, had, "") + return + case in.BusNode == "": + why = "no node runs the bus's module, so the list is sealed to nobody" + case key == "": + why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" + case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode: + object, err := delivery.SealUserList(list, key) + if err == nil { + err = p.Objects.PutObject(delivery.UserListObjectName(digest), object) + } + if err != nil { + res.Errors = append(res.Errors, "the sealed user list could not be put: "+err.Error()) + res.Sealed = sealedSays(cur, had, "") + return + } + if cur.Digest != digest { + next.Revision++ + } + next.Digest, next.Object, next.SealedTo, next.Node = digest, delivery.UserListObjectName(digest), delivery.SealingKeyID(key), in.BusNode + } + next.Users, next.LeftOut = users, leftOut + if next.Digest == "" { + next.Users, next.LeftOut = 0, nil + } + if had && sameSealed(cur, next) { + res.Sealed = sealedSays(cur, had, why) + return + } + next.At = p.now() + data, err := delivery.EncodeGrantsUserList(next) + if err == nil { + _, err = p.Store.Put(StateUserList, KeySealed, data) + } + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealed, err)) + res.Sealed = sealedSays(cur, had, "") + return + } + // The object the key named before is taken away once the key names the new one: the bucket keeps the list in force. + if had && cur.Object != "" && cur.Object != next.Object { + if err := p.Objects.DeleteObject(cur.Object); err != nil { + res.Errors = append(res.Errors, "the sealed list before could not be taken away: "+err.Error()) + } + } + res.Sealed = sealedSays(next, true, why) + p.say(res, EventUserList, map[string]any{"digest": next.Digest, "revision": next.Revision, "users": next.Users, + "left_out": len(next.LeftOut), "follow": next.Follow, "sealed_to": next.Node}) +} + +func sameSealed(a, b delivery.GrantsUserList) bool { + a.At, b.At = b.At, a.At + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// SealedState is what a pass says of the sealed list: the key as written, and why nothing was written when nothing was. +type SealedState struct { + Written *delivery.GrantsUserList `json:"written,omitempty"` + Why string `json:"why,omitempty"` +} + +func sealedSays(g delivery.GrantsUserList, had bool, why string) *SealedState { + s := &SealedState{Why: why} + if had { + s.Written = &g + } + return s +} + +// syncFollow writes the switch into the key at once, so the bus node turns on the operator's yes, not at the next pass. +func (p *Part) syncFollow(state string) error { + cur, _, had := p.Sealed() + if had && cur.Follow == followOf(state) { + return nil + } + if !had { + cur = delivery.GrantsUserList{} + } + cur.Follow, cur.At = followOf(state), p.now() + data, err := delivery.EncodeGrantsUserList(cur) + if err != nil { + return err + } + _, err = p.Store.Put(StateUserList, KeySealed, data) + return err +} + +// Effect is where the switch stands on the bus node: "switched" only once the bus node's loaded list is the grants part's +// written one (its source the part's key, its digest the written digest); otherwise what it waits for. +func (p *Part) Effect(state string, loaded *delivery.UserListWritten) string { + g, _, had := p.Sealed() + fromGrants := loaded != nil && loaded.Source == delivery.GrantsUserListSource + switch { + case state == TookOver && had && fromGrants && g.Digest != "" && loaded.Digest == g.Digest && loaded.Error == "": + return "switched: the bus node loaded the grants part's list " + short(g.Digest) + " (revision " + fmt.Sprint(g.Revision) + ")" + case state == TookOver && !had || state == TookOver && g.Digest == "": + return "recorded, not in effect: the grants part has written no sealed list for the bus node yet" + case state == TookOver && loaded == nil: + return "recorded, not in effect: the bus module's have names no loaded list" + case state == TookOver && loaded.Error != "": + return "recorded, not in effect: the bus node could not load the grants part's list: " + loaded.Error + case state == TookOver: + return "recorded, not in effect yet: the bus node loaded " + short(loaded.Digest) + " from " + loaded.Source + ", the grants part wrote " + short(g.Digest) + case fromGrants: + return "handed back, not in effect yet: the bus node still follows the grants part's list" + } + return "shadow: the bus node follows the controller's list" +} + +func short(d string) string { + d = strings.TrimPrefix(d, "sha256:") + if len(d) > 12 { + d = d[:12] + } + return d +} + +// followReasons is what a take-over waits for on the sealed list itself: a list written for the bus node, of the list +// composed now, and the loaded list granting the bus node its read (the controller's composer carries ADR 0547 clause 7's +// grants from the operator's second exception; until its delivery the bus node could not read the first list). +func (p *Part) followReasons(res Result, in Inputs) []string { + var out []string + g, _, had := p.Sealed() + switch { + case !had || g.Digest == "": + why := "no list is written" + if res.Sealed != nil && res.Sealed.Why != "" { + why = res.Sealed.Why + } + out = append(out, "the grants part has written no sealed list for the bus node: "+why) + case in.Loaded == nil || in.Loaded.Key == "" || g.SealedTo != delivery.SealingKeyID(in.Loaded.Key): + out = append(out, "the sealed list is not sealed to the key the bus node says now") + } + if in.BusNode != "" { + node := "node." + in.BusNode + for _, d := range res.UserList.Compare.Differ { + if d.User == node && slices.ContainsFunc(d.PublishAdded, func(s string) bool { return strings.Contains(s, delivery.GrantsUserListBucket) }) { + out = append(out, "the loaded list does not grant "+node+" reading the grants part's sealed list (ADR 0547 clause 7): "+ + "the controller's composer carries it from its delivery of the operator's second exception") + } + } + } + return out +} diff --git a/modules/mesh-grants/internal/grants/takeover.go b/modules/mesh-grants/internal/grants/takeover.go index 4fe6b41..6ee843d 100644 --- a/modules/mesh-grants/internal/grants/takeover.go +++ b/modules/mesh-grants/internal/grants/takeover.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // The switch between the controller's user list and the grants part's (ADR 0547 clause 8). The operator's word is a @@ -75,6 +76,8 @@ type Asker interface { Where(id string) (string, *asks.Warrant, error) // Take hands a warrant to act once (a warrant the process missed is caught up through it). Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) + // Cancel takes an ask back at the router, so a copy the operator may still see ends there too. + Cancel(id string) error } // askMargin is how long past an ask's expiry the router is given to say it ended, before the part closes it itself; @@ -203,6 +206,11 @@ func (p *Part) RaiseRequested() error { if p.Forget != nil { _ = p.Forget(r.ID) } + // A publish that timed out may still have reached the router: taken back there, so a yes to it cannot land on an + // ask the part has forgotten. + if cerr := p.Asker.Cancel(r.ID); cerr != nil && p.Logf != nil { + p.Logf("the ask %s that could not be published was not taken back at the router: %v", r.ID, cerr) + } if s, rerr := p.readSwitch(); rerr == nil { _ = p.finish(s, Act{Act: r.Act, By: "operator", At: p.now(), Ask: r.ID, Said: "refused", Refused: "the operator could not be asked: " + err.Error()}) } @@ -242,6 +250,10 @@ func (p *Part) Reconcile() error { } if w != nil { h, herr := p.Asker.Take(*w, p.Warranted) + if h == "" && herr != nil { + // The part's own record of the ask could not be reached: nothing was taken, and the next pass tries again. + return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", s.Open, herr) + } p.Said(h, *w, herr) if cur, rerr := p.readSwitch(); rerr == nil && cur.Open == s.Open { _ = p.finish(cur, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(w.Outcome)}) @@ -272,11 +284,12 @@ func actOf(id string) string { // Following says whether the bus node has loaded the grants part's list: "yes" when its loaded digest equals the part's, // "not yet" when it loaded another, "unknown" when its have says none. func (p *Part) Following() string { - st := p.Last().UserList - if st.Loaded == nil || st.Digest == "" { - return "unknown: the bus module's have names no loaded list" + loaded := ReadInputs(p.Read, &p.cache).Loaded + g, _, had := p.Sealed() + if loaded == nil || !had || g.Digest == "" { + return "unknown: the bus module's have names no loaded list, or the grants part wrote none" } - if st.Loaded.Wanted == st.Digest || st.Loaded.Digest == st.Digest { + if loaded.Source == delivery.GrantsUserListSource && loaded.Digest == g.Digest { return "yes" } return "not yet: the bus node loaded another list" @@ -342,15 +355,25 @@ func (p *Part) Warranted(o asks.Option, w asks.Warrant) error { } else { s.State = Shadow } + if err := p.finish(s, a); err != nil { + return err + } + // The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8). + if err := p.syncFollow(s.State); err != nil { + return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err) + } if p.Emit != nil { _ = p.Emit.Emit(EventUserList, map[string]any{"switch": s.State}) } - return p.finish(s, a) + return nil } // Said is the process's note of an ask that ended with no yes or no (expired, or refused by the router): the switch // stays, and the ask is closed so the operator can be asked again. -func (p *Part) Said(h asker.Handled, w asks.Warrant, _ error) { +func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) { + if err != nil && p.Logf != nil { + p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err) + } s, err := p.readSwitch() if err != nil || s.Open != w.Ask { return @@ -383,6 +406,7 @@ func (p *Part) Decide() []string { reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list: %s", len(d), strings.Join(d, ", "))) } } + reasons = append(reasons, p.followReasons(res, ReadInputs(p.Read, &p.cache))...) cr, gone := res.UserList.Credentials, res.UserList.Compare.Gone var unhashed []string for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) { @@ -448,8 +472,18 @@ func (p *Part) TakeOverState() (any, error) { if err != nil { return nil, err } + loaded := ReadInputs(p.Read, &p.cache).Loaded out := map[string]any{"state": s.State, "open": s.Open, "acts": s.Acts, "bus_node_loaded_grants_list": p.Following(), - "says": "a recorded switch takes effect when the bus node loads the grants part's list; bus_node_loaded_grants_list says whether it has (after a hand-back the controller's list is meant, and it then reads not yet)"} + "effect": p.Effect(s.State, loaded), + "says": "a recorded switch takes effect when the bus node loads the grants part's list; effect says switched only once the bus node's loaded digest is the grants part's written one"} + if g, rev, had := p.Sealed(); had { + out["written"] = map[string]any{"digest": g.Digest, "revision": g.Revision, "key_revision": rev, "follow": g.Follow, + "users": g.Users, "left_out": g.LeftOut, "sealed_to": g.Node} + } + if loaded != nil { + out["loaded"] = map[string]any{"source": loaded.Source, "digest": loaded.Digest, "revision": loaded.Revision, + "users": loaded.Users, "follows": loaded.Follows, "at": loaded.At, "error": loaded.Error} + } if raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest); err == nil && found { out["request"] = json.RawMessage(raw) }