diff --git a/modules/mesh-grants/cmd/mesh-grants/asking.go b/modules/mesh-grants/cmd/mesh-grants/asking.go index 15db062..3095d96 100644 --- a/modules/mesh-grants/cmd/mesh-grants/asking.go +++ b/modules/mesh-grants/cmd/mesh-grants/asking.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "time" "github.com/nats-io/nats.go" @@ -40,8 +41,10 @@ func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id stri ctx, cancel := context.WithTimeout(ctx, recordWithin) defer cancel() msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_asks.$KV.asks."+module+"."+id, nil) + // No responders is the bucket out of reach for a moment, never "no record": read as none, a live ask would be + // closed by Reconcile a minute later and the operator's yes lost. if errors.Is(err, nats.ErrNoResponders) { - return nil, nil + return nil, fmt.Errorf("the router's record of %s cannot be reached: %w", id, err) } if err != nil { return nil, err diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index 365de24..d971edc 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -7,8 +7,9 @@ // module placed since cannot serve its seat or reach a new state. Each is published on the assignments stream only when // it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is // created** (the operator, 2026-10-26 08:54; issue 1790), once per manifest, with the manifest's limits, on the event -// that brought the manifest; one the bus holds is never changed. **The user list, the rest of the assertions and the -// grants stay in shadow** until +// that brought the manifest; one the bus holds is never changed. **The user list is written sealed to the bus node** +// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows only +// from take-over. **The rest of the assertions and the grants stay in shadow** until // the operator's take-over, asked on their channel and run on their yes after its refusals: they are composed, compared with what the bus loaded // and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect. // @@ -116,7 +117,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } bus := newBusReader(ctx, js) f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn}, - Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}} + Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, + Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf} f.Watch = bus // The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on // the warrant once (mesh-sdk asker), after its refusals. @@ -149,7 +151,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) return err } - logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; the user list, grants and any change to a held stream in shadow", account.User) + logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; and writing the user list sealed to the bus node, which the bus node follows from take-over; grants and any change to a held stream in shadow", account.User) f.Run(ctx) return nil } diff --git a/modules/mesh-grants/cmd/mesh-grants/sealed.go b/modules/mesh-grants/cmd/mesh-grants/sealed.go new file mode 100644 index 0000000..1c8156e --- /dev/null +++ b/modules/mesh-grants/cmd/mesh-grants/sealed.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// sealedObjects is the part's object bucket of user lists sealed to the bus node (novox/hq ADR 0547 clause 7), created on +// its first put: the composer grants the part its create, its puts and deletes, and nothing of another bucket. +type sealedObjects struct { + ctx context.Context + js jetstream.JetStream +} + +func (o sealedObjects) store(ctx context.Context) (jetstream.ObjectStore, error) { + s, err := o.js.ObjectStore(ctx, delivery.GrantsUserListObjects) + if errors.Is(err, jetstream.ErrBucketNotFound) { + s, err = o.js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: delivery.GrantsUserListObjects, + Description: "the bus's user list composed by the grants part, each sealed to the bus node's key under the hex of its digest (novox/hq ADR 0547 clause 7): written by mesh-grants alone, read by name by the node-engine of the node holding mesh-broker", + Storage: jetstream.FileStorage, MaxBytes: 64 << 20}) + } + if err != nil { + return nil, fmt.Errorf("the object bucket %s: %w", delivery.GrantsUserListObjects, err) + } + return s, nil +} + +func (o sealedObjects) PutObject(name string, data []byte) error { + ctx, cancel := context.WithTimeout(o.ctx, requestWithin) + defer cancel() + s, err := o.store(ctx) + if err != nil { + return err + } + _, err = s.PutBytes(ctx, name, data) + return err +} + +func (o sealedObjects) DeleteObject(name string) error { + ctx, cancel := context.WithTimeout(o.ctx, requestWithin) + defer cancel() + s, err := o.store(ctx) + if err != nil { + return err + } + if err := s.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 393b876..6ef66ab 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 919fd4e..9c3f761 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896 h1:5rHfru5NJ7SK20QOt8YUJSglcn1gsjtLc9AGmWYGnRQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026080443-136fa8b2a896/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd h1:ef6y09csev6xY/suFrIFGUXbgvDA29kummFtHcIb3B8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 2d7a5a2..e2f8a60 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "mesh-grants/internal/grants" ) @@ -344,6 +345,7 @@ func TestGrants(t *testing.T) { createOnly := filepath.Join("cmd", "mesh-grants", "create.go") // And the operator's asks and conditions, on the seat operator-channel's subjects, in asking.go (ADR 0547 clause 8). asksOnly := filepath.Join("cmd", "mesh-grants", "asking.go") + sealedOnly := filepath.Join("cmd", "mesh-grants", "sealed.go") files := 0 for _, root := range []string{"cmd", "internal"} { err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { @@ -364,6 +366,13 @@ func TestGrants(t *testing.T) { if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) { t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n) } + // The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7). + if path == sealedOnly { + if strings.Count(string(src), "CreateObjectStore(") != 1 || !strings.Contains(string(src), "delivery.GrantsUserListObjects") { + t.Errorf("sealed.go creates other than the part's one object bucket") + } + src = []byte(strings.ReplaceAll(string(src), "CreateObjectStore", "")) + } for _, f := range forbidden { if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the part holds no timer and creates or asserts nothing", path, f) @@ -730,6 +739,30 @@ func TestGrants(t *testing.T) { b.loaded = shadowOf(t, b) b.streams = []map[string]any{{"name": "KV_mod-x_kept", "subjects": []string{"$KV.mod-x_kept.>"}, "retention": "limits", "max_per_subject": 3}} p := partOn(b) + // The bus node says its key in its have of the bus module; the process puts the sealed list (ADR 0547 clause 7). + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + withKey := func(w delivery.UserListWritten) { + w.Key = busKey.PublicKey + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, UserList: &w}) + } + loadedBefore := delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 6} + withKey(loadedBefore) + objs := &fakeObjects{} + p.Objects = objs + open := func(g delivery.GrantsUserList) string { + t.Helper() + list, err := delivery.OpenUserList(objs.objects[g.Object], g.Digest, func(bx seal.Box) ([]byte, error) { + out, err := seal.Open(bx, busKey.PrivateKey) + return []byte(out), err + }) + if err != nil { + t.Fatalf("the sealed list does not open with the bus node's key: %v", err) + } + return string(list) + } got, _ := p.UserList() if !strings.Contains(fmt.Sprint(got), "nothing composed yet") { @@ -739,6 +772,20 @@ func TestGrants(t *testing.T) { if !first.Written || first.Revision != 1 || !first.UserList.Compare.Equal() || first.UserList.Users != 6 { t.Fatalf("first composition = %+v", first) } + // Written in shadow, the bus node told to follow the controller's list; every user left out while the vault holds + // no hash; a tool's part (no object bucket) writes none. + if g, _, had := p.Sealed(); !had || g.Follow != delivery.FollowController || g.Revision != 1 || g.Node != "a" || + g.Users != 0 || len(g.LeftOut) != 6 || len(objs.objects) != 1 || !strings.Contains(open(g), "accounts {") { + t.Errorf("the sealed list in shadow = %+v, %d objects", g, len(objs.objects)) + } + if _, _, had := partOn(fixture()).Sealed(); had { + t.Error("a part read before any pass holds a sealed list") + } + if tp := partOn(fixture()); tp.Compose().Sealed == nil { + t.Error("a tool's pass does not say the sealed list") + } else if _, _, had := tp.Sealed(); had { + t.Error("a tool's part wrote the sealed list") + } if len(first.Published) != 3 || len(b.publishes) != 3 { t.Errorf("the first composition published %v, want the three assignments' memberships", first.Published) } @@ -889,6 +936,26 @@ func TestGrants(t *testing.T) { if row.outcome == "recorded" && !strings.Contains(fmt.Sprint(p.TakeOverState()), "bus_node_loaded_grants_list:not yet") { t.Errorf("the state claims the bus node follows with no loaded list: %v", fmt.Sprint(p.TakeOverState())) } + // The key the bus node follows turns with the switch at once, and the state says switched only once the bus + // node loaded the grants part's written list. + g, _, _ := p.Sealed() + if want := map[string]string{grants.TookOver: delivery.FollowGrants, grants.Shadow: delivery.FollowController}[p.SwitchState()]; g.Follow != want { + t.Errorf("the key says follow %s with the switch %s", g.Follow, p.SwitchState()) + } + effect := func() string { st, _ := p.TakeOverState(); return fmt.Sprint(st.(map[string]any)["effect"]) } + if row.outcome == "recorded" && row.act == "take-over" { + if e := effect(); strings.HasPrefix(e, "switched") || !strings.Contains(e, "not in effect") { + t.Errorf("effect before the bus node loaded = %s", e) + } + if users := strings.Count(open(g), "{ user: "); users != g.Users || g.Users == 0 || len(g.LeftOut) != 0 { + t.Errorf("the sealed list holds %d users, the key says %d, left out %v", users, g.Users, g.LeftOut) + } + withKey(delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Wanted: g.Digest, Digest: g.Digest, Users: g.Users, Follows: delivery.FollowGrants}) + if e := effect(); !strings.HasPrefix(e, "switched") { + t.Errorf("effect once the bus node loaded the written list = %s", e) + } + withKey(loadedBefore) + } }) } t.Run("hand-back in shadow is refused at once, with nothing asked", func(t *testing.T) { @@ -943,10 +1010,25 @@ func TestGrants(t *testing.T) { id = asked("take-over") p.Compose() p.Compose() - if l := last(); l.Ask != id || l.Said != "refused" || !slices.Contains(forgot, id) { - t.Errorf("a failed ask stays open: %+v, forgot %v", l, forgot) + if l := last(); l.Ask != id || l.Said != "refused" || !slices.Contains(forgot, id) || !slices.Contains(fa.cancels, id) { + t.Errorf("a failed ask stays open, or is not taken back at the router: %+v, forgot %v, cancelled %v", l, forgot, fa.cancels) } fa.fail = false + // An answer the part's own record cannot take yet (its state out of reach) leaves the ask open, closed by + // nothing, and is taken at a later pass. + id = asked("take-over") + p.Compose() + fa.wheres[id] = &asks.Warrant{Ask: id, Outcome: asks.OutcomeChosen, Option: "no"} + fa.takeErr = errors.New("nats: no responders available for request") + p.Compose() + if l := last(); l.Ask != id || l.Said != "asked" { + t.Errorf("an answer not taken closed the ask: %+v", l) + } + fa.takeErr = nil + p.Compose() + if l := last(); l.Ask != id || l.Said != "declined" { + t.Errorf("the answer was not taken once its record could be reached: %+v", l) + } asked("take-over") }) @@ -1259,11 +1341,29 @@ func TestGrants(t *testing.T) { } type fakeAsker struct { - asks []asks.Ask - wheres map[string]*asks.Warrant // the router's warrants, by ask - fail bool + asks []asks.Ask + wheres map[string]*asks.Warrant // the router's warrants, by ask + fail bool + cancels []string + // takeErr makes Take fail as the asker does when the part's own record cannot be reached: no handling, an error. + takeErr error } +func (f *fakeAsker) Cancel(id string) error { f.cancels = append(f.cancels, id); return nil } + +// fakeObjects is the part's object bucket of sealed lists. +type fakeObjects struct{ objects map[string][]byte } + +func (o *fakeObjects) PutObject(name string, data []byte) error { + if o.objects == nil { + o.objects = map[string][]byte{} + } + o.objects[name] = data + return nil +} + +func (o *fakeObjects) DeleteObject(name string) error { delete(o.objects, name); return nil } + func (f *fakeAsker) Ask(a asks.Ask) error { if f.fail { return errors.New("the bus refused the publish") @@ -1276,6 +1376,9 @@ func (f *fakeAsker) Where(id string) (string, *asks.Warrant, error) { return "op // Take hands the warrant to act as the SDK's asker does for one that chose, and records any other. func (f *fakeAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) { + if f.takeErr != nil { + return "", f.takeErr + } if w.Outcome != asks.OutcomeChosen { return asker.Recorded, nil } diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index f864fcc..0c074e0 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -101,9 +101,11 @@ type Result struct { Refused []string `json:"refused,omitempty"` // RootNotFree is each machine the composition needed a root-free word for and did not get one, `: not // free, `: a verified sender is composed as from no root-free machine there (issue 1945). - RootNotFree []string `json:"root_not_free,omitempty"` - Errors []string `json:"errors,omitempty"` - PublishedOrAsserted string `json:"published_or_asserted"` + RootNotFree []string `json:"root_not_free,omitempty"` + // Sealed is the list sealed to the bus node as the key `user-list.sealed` names it (ADR 0547 clause 7). + Sealed *SealedState `json:"sealed,omitempty"` + Errors []string `json:"errors,omitempty"` + PublishedOrAsserted string `json:"published_or_asserted"` } // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. @@ -122,6 +124,10 @@ type Part struct { // Asker asks the operator (take-over, hand-back): the process's client of the router; nil in the tools, which write // the request and let the process ask it. Asker Asker + // Logf logs what no state or condition says (an error of acting on an answer): the process's; nil in the tools. + Logf func(string, ...any) + // Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools. + Objects Objects // Say says a condition to the operator: the process's; nil in the tools. Say func(asks.Condition) error // Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix. @@ -219,6 +225,12 @@ func (p *Part) Compose() Result { } } } + // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). + if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil { + res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error()) + } else { + p.writeSealed(&res, list, n, leftOut, in) + } ms := Memberships(recs) res.Memberships = len(ms) for _, key := range sortedKeys(ms) { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go new file mode 100644 index 0000000..82c047a --- /dev/null +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -0,0 +1,231 @@ +package grants + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **The list the bus node loads from take-over** (novox/hq ADR 0547 clause 7, issue 1945). Each pass composes the bus's +// user list with the vault's hashes (the accounts block, buscompose.ComposeAccounts, as the bus module's file holds it), +// seals it to the bus node's key (the key the node says in its have of the bus module) and puts it as one object of the +// part's object bucket, named by its digest; then writes the key `user-list.sealed` naming it, with the revision, the user +// count, the users left out for want of a hash, and which list the bus node follows (the switch, clause 8). The object +// and the key are written in shadow too: the bus node reads the key and follows the controller's list until the key says +// FollowGrants, so a take-over has a list to load at once. Only the process writes them (Objects); a tool asks it. + +// KeySealed is the key of the state user-list the bus node reads (delivery.GrantsUserListKey). +const KeySealed = delivery.GrantsUserListKey + +// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools. +type Objects interface { + PutObject(name string, data []byte) error + DeleteObject(name string) error +} + +// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault, and the users left out +// for want of one. Pure. +func ComposeList(r buscompose.Records, hashes map[string]HeldHash) ([]byte, int, []string, error) { + principals, err := buscompose.Users(r) + if err != nil { + return nil, 0, nil, err + } + held := map[string]string{} + for user, h := range hashes { + held[user] = h.Hash + } + filled, missing := buscompose.WithPasswords(principals, held) + list, err := buscompose.ComposeAccounts(filled) + if err != nil { + return nil, 0, nil, err + } + slices.Sort(missing) + return []byte(list), len(filled), missing, nil +} + +// Sealed is the part's key `user-list.sealed` as written, and whether there is one. +func (p *Part) Sealed() (delivery.GrantsUserList, uint64, bool) { + raw, rev, found, err := p.Store.Get(StateUserList, KeySealed) + if err != nil || !found { + return delivery.GrantsUserList{}, 0, false + } + g, err := delivery.DecodeGrantsUserList(raw) + if err != nil { + return delivery.GrantsUserList{}, 0, false + } + return g, rev, true +} + +// followOf is which list the switch says the bus node follows. +func followOf(state string) string { + if state == TookOver { + return delivery.FollowGrants + } + return delivery.FollowController +} + +// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in +// res what it wrote, or why it wrote nothing. +func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs) { + cur, _, had := p.Sealed() + next := cur + next.Follow = followOf(p.SwitchState()) + key := "" + if in.Loaded != nil { + key = in.Loaded.Key + } + digest := delivery.UserListDigest(list) + why := "" + switch { + case p.Objects == nil: + // The tools: the process writes; the follow alone may change here, from a tool's view of the switch, and is not. + res.Sealed = sealedSays(cur, had, "") + return + case in.BusNode == "": + why = "no node runs the bus's module, so the list is sealed to nobody" + case key == "": + why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" + case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode: + object, err := delivery.SealUserList(list, key) + if err == nil { + err = p.Objects.PutObject(delivery.UserListObjectName(digest), object) + } + if err != nil { + res.Errors = append(res.Errors, "the sealed user list could not be put: "+err.Error()) + res.Sealed = sealedSays(cur, had, "") + return + } + if cur.Digest != digest { + next.Revision++ + } + next.Digest, next.Object, next.SealedTo, next.Node = digest, delivery.UserListObjectName(digest), delivery.SealingKeyID(key), in.BusNode + } + next.Users, next.LeftOut = users, leftOut + if next.Digest == "" { + next.Users, next.LeftOut = 0, nil + } + if had && sameSealed(cur, next) { + res.Sealed = sealedSays(cur, had, why) + return + } + next.At = p.now() + data, err := delivery.EncodeGrantsUserList(next) + if err == nil { + _, err = p.Store.Put(StateUserList, KeySealed, data) + } + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealed, err)) + res.Sealed = sealedSays(cur, had, "") + return + } + // The object the key named before is taken away once the key names the new one: the bucket keeps the list in force. + if had && cur.Object != "" && cur.Object != next.Object { + if err := p.Objects.DeleteObject(cur.Object); err != nil { + res.Errors = append(res.Errors, "the sealed list before could not be taken away: "+err.Error()) + } + } + res.Sealed = sealedSays(next, true, why) + p.say(res, EventUserList, map[string]any{"digest": next.Digest, "revision": next.Revision, "users": next.Users, + "left_out": len(next.LeftOut), "follow": next.Follow, "sealed_to": next.Node}) +} + +func sameSealed(a, b delivery.GrantsUserList) bool { + a.At, b.At = b.At, a.At + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// SealedState is what a pass says of the sealed list: the key as written, and why nothing was written when nothing was. +type SealedState struct { + Written *delivery.GrantsUserList `json:"written,omitempty"` + Why string `json:"why,omitempty"` +} + +func sealedSays(g delivery.GrantsUserList, had bool, why string) *SealedState { + s := &SealedState{Why: why} + if had { + s.Written = &g + } + return s +} + +// syncFollow writes the switch into the key at once, so the bus node turns on the operator's yes, not at the next pass. +func (p *Part) syncFollow(state string) error { + cur, _, had := p.Sealed() + if had && cur.Follow == followOf(state) { + return nil + } + if !had { + cur = delivery.GrantsUserList{} + } + cur.Follow, cur.At = followOf(state), p.now() + data, err := delivery.EncodeGrantsUserList(cur) + if err != nil { + return err + } + _, err = p.Store.Put(StateUserList, KeySealed, data) + return err +} + +// Effect is where the switch stands on the bus node: "switched" only once the bus node's loaded list is the grants part's +// written one (its source the part's key, its digest the written digest); otherwise what it waits for. +func (p *Part) Effect(state string, loaded *delivery.UserListWritten) string { + g, _, had := p.Sealed() + fromGrants := loaded != nil && loaded.Source == delivery.GrantsUserListSource + switch { + case state == TookOver && had && fromGrants && g.Digest != "" && loaded.Digest == g.Digest && loaded.Error == "": + return "switched: the bus node loaded the grants part's list " + short(g.Digest) + " (revision " + fmt.Sprint(g.Revision) + ")" + case state == TookOver && !had || state == TookOver && g.Digest == "": + return "recorded, not in effect: the grants part has written no sealed list for the bus node yet" + case state == TookOver && loaded == nil: + return "recorded, not in effect: the bus module's have names no loaded list" + case state == TookOver && loaded.Error != "": + return "recorded, not in effect: the bus node could not load the grants part's list: " + loaded.Error + case state == TookOver: + return "recorded, not in effect yet: the bus node loaded " + short(loaded.Digest) + " from " + loaded.Source + ", the grants part wrote " + short(g.Digest) + case fromGrants: + return "handed back, not in effect yet: the bus node still follows the grants part's list" + } + return "shadow: the bus node follows the controller's list" +} + +func short(d string) string { + d = strings.TrimPrefix(d, "sha256:") + if len(d) > 12 { + d = d[:12] + } + return d +} + +// followReasons is what a take-over waits for on the sealed list itself: a list written for the bus node, of the list +// composed now, and the loaded list granting the bus node its read (the controller's composer carries ADR 0547 clause 7's +// grants from the operator's second exception; until its delivery the bus node could not read the first list). +func (p *Part) followReasons(res Result, in Inputs) []string { + var out []string + g, _, had := p.Sealed() + switch { + case !had || g.Digest == "": + why := "no list is written" + if res.Sealed != nil && res.Sealed.Why != "" { + why = res.Sealed.Why + } + out = append(out, "the grants part has written no sealed list for the bus node: "+why) + case in.Loaded == nil || in.Loaded.Key == "" || g.SealedTo != delivery.SealingKeyID(in.Loaded.Key): + out = append(out, "the sealed list is not sealed to the key the bus node says now") + } + if in.BusNode != "" { + node := "node." + in.BusNode + for _, d := range res.UserList.Compare.Differ { + if d.User == node && slices.ContainsFunc(d.PublishAdded, func(s string) bool { return strings.Contains(s, delivery.GrantsUserListBucket) }) { + out = append(out, "the loaded list does not grant "+node+" reading the grants part's sealed list (ADR 0547 clause 7): "+ + "the controller's composer carries it from its delivery of the operator's second exception") + } + } + } + return out +} diff --git a/modules/mesh-grants/internal/grants/takeover.go b/modules/mesh-grants/internal/grants/takeover.go index 4fe6b41..6ee843d 100644 --- a/modules/mesh-grants/internal/grants/takeover.go +++ b/modules/mesh-grants/internal/grants/takeover.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // The switch between the controller's user list and the grants part's (ADR 0547 clause 8). The operator's word is a @@ -75,6 +76,8 @@ type Asker interface { Where(id string) (string, *asks.Warrant, error) // Take hands a warrant to act once (a warrant the process missed is caught up through it). Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) + // Cancel takes an ask back at the router, so a copy the operator may still see ends there too. + Cancel(id string) error } // askMargin is how long past an ask's expiry the router is given to say it ended, before the part closes it itself; @@ -203,6 +206,11 @@ func (p *Part) RaiseRequested() error { if p.Forget != nil { _ = p.Forget(r.ID) } + // A publish that timed out may still have reached the router: taken back there, so a yes to it cannot land on an + // ask the part has forgotten. + if cerr := p.Asker.Cancel(r.ID); cerr != nil && p.Logf != nil { + p.Logf("the ask %s that could not be published was not taken back at the router: %v", r.ID, cerr) + } if s, rerr := p.readSwitch(); rerr == nil { _ = p.finish(s, Act{Act: r.Act, By: "operator", At: p.now(), Ask: r.ID, Said: "refused", Refused: "the operator could not be asked: " + err.Error()}) } @@ -242,6 +250,10 @@ func (p *Part) Reconcile() error { } if w != nil { h, herr := p.Asker.Take(*w, p.Warranted) + if h == "" && herr != nil { + // The part's own record of the ask could not be reached: nothing was taken, and the next pass tries again. + return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", s.Open, herr) + } p.Said(h, *w, herr) if cur, rerr := p.readSwitch(); rerr == nil && cur.Open == s.Open { _ = p.finish(cur, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(w.Outcome)}) @@ -272,11 +284,12 @@ func actOf(id string) string { // Following says whether the bus node has loaded the grants part's list: "yes" when its loaded digest equals the part's, // "not yet" when it loaded another, "unknown" when its have says none. func (p *Part) Following() string { - st := p.Last().UserList - if st.Loaded == nil || st.Digest == "" { - return "unknown: the bus module's have names no loaded list" + loaded := ReadInputs(p.Read, &p.cache).Loaded + g, _, had := p.Sealed() + if loaded == nil || !had || g.Digest == "" { + return "unknown: the bus module's have names no loaded list, or the grants part wrote none" } - if st.Loaded.Wanted == st.Digest || st.Loaded.Digest == st.Digest { + if loaded.Source == delivery.GrantsUserListSource && loaded.Digest == g.Digest { return "yes" } return "not yet: the bus node loaded another list" @@ -342,15 +355,25 @@ func (p *Part) Warranted(o asks.Option, w asks.Warrant) error { } else { s.State = Shadow } + if err := p.finish(s, a); err != nil { + return err + } + // The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8). + if err := p.syncFollow(s.State); err != nil { + return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err) + } if p.Emit != nil { _ = p.Emit.Emit(EventUserList, map[string]any{"switch": s.State}) } - return p.finish(s, a) + return nil } // Said is the process's note of an ask that ended with no yes or no (expired, or refused by the router): the switch // stays, and the ask is closed so the operator can be asked again. -func (p *Part) Said(h asker.Handled, w asks.Warrant, _ error) { +func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) { + if err != nil && p.Logf != nil { + p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err) + } s, err := p.readSwitch() if err != nil || s.Open != w.Ask { return @@ -383,6 +406,7 @@ func (p *Part) Decide() []string { reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list: %s", len(d), strings.Join(d, ", "))) } } + reasons = append(reasons, p.followReasons(res, ReadInputs(p.Read, &p.cache))...) cr, gone := res.UserList.Credentials, res.UserList.Compare.Gone var unhashed []string for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) { @@ -448,8 +472,18 @@ func (p *Part) TakeOverState() (any, error) { if err != nil { return nil, err } + loaded := ReadInputs(p.Read, &p.cache).Loaded out := map[string]any{"state": s.State, "open": s.Open, "acts": s.Acts, "bus_node_loaded_grants_list": p.Following(), - "says": "a recorded switch takes effect when the bus node loads the grants part's list; bus_node_loaded_grants_list says whether it has (after a hand-back the controller's list is meant, and it then reads not yet)"} + "effect": p.Effect(s.State, loaded), + "says": "a recorded switch takes effect when the bus node loads the grants part's list; effect says switched only once the bus node's loaded digest is the grants part's written one"} + if g, rev, had := p.Sealed(); had { + out["written"] = map[string]any{"digest": g.Digest, "revision": g.Revision, "key_revision": rev, "follow": g.Follow, + "users": g.Users, "left_out": g.LeftOut, "sealed_to": g.Node} + } + if loaded != nil { + out["loaded"] = map[string]any{"source": loaded.Source, "digest": loaded.Digest, "revision": loaded.Revision, + "users": loaded.Users, "follows": loaded.Follows, "at": loaded.At, "error": loaded.Error} + } if raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest); err == nil && found { out["request"] = json.RawMessage(raw) }