// Package userlist is the bridge of novox/hq ADR 0547 §8: the node agent of the node holding mesh-broker applies the // bus's user list from the newest composition of the bus module the controller writes for its node, alone and with no // want, so a new account is in force on the bus without anybody's hand. // // **Why it exists.** The list reached the bus only through a want of the bus module, and ADR 0539 §15 bars the assigner // from writing one: on 2026-10-25 it stood from 11:23 to 20:50 and every new account was refused until the operator put // it back by hand (issue 1945). The bus module's entrypoint already reloads the server in place when the file changes; // what was missing is a writer of the file that does not wait on a want. // // **It watches, it never polls.** It follows the node's composition bucket. Each composition is verified as the agent // verifies it (signed by the controller's or the operator's key) before anything of it is read. Of those that carry the // bus's user list (a file resource `bus-users`), the newest by revision is the source; its list is written alone // (apply.WriteAlone) when it differs from the file on the machine, and what was written is reported in the bus // module's have. It looks again after every apply of the bus module, since an apply of an older composition writes an // older list, and when asked (Again). // // It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list. // // **It hands the loaded list's hashes to the vault** (ADR 0547 clause 9, issue 1945): once the vault's key is known // (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to // that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only // when the list on the machine or the vault's key changes. // // **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose // compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says // FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked // against the key's digest, and written to the same file (mode 0600) only when the file's digest differs; otherwise the // controller's compositions are the source, as before. Never both. The have says which it follows and this node's key, // which the grants part seals to. package userlist import ( "context" "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "regexp" "slices" "strings" "sync" "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/agent" ) // ResourceID is the id the composer gives the bus's user list (mesh-sdk catalogue.BusUsersID); in a composition it may // carry its module before it, `.bus-users`. const ResourceID = "bus-users" // Written is what the follower last did: the source it took the list from, the list's digest there, the file's digest // on the machine after the write, how many users the list holds, when, and why a write failed. type Written = delivery.UserListWritten // Env is everything the follower needs of the node. type Env struct { Node string Composed agent.Watcher // Trusted is the keys a composition may be signed by, as the agent trusts them. Trusted func() []ed25519.PublicKey // Resolve is the list a user-list resource holds: its content, or the object its digest names, fetched and // verified (the controller sends a file over 64 KiB by digest, ADR 0486 §1, and the live list is over it). Resolve func(ctx context.Context, resource json.RawMessage) ([]byte, error) // OnDisk is what the resource's file holds on the machine now; nil when there is none. OnDisk func(resource json.RawMessage) ([]byte, error) // Write writes the user list's file resource of a module's composition alone, with body as its content, when it // differs from the file on the machine, and answers the digest of the file there after it (`sha256:`) and // whether it changed. Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error) // Report says what was written, in the module's have. Report func(ctx context.Context, module string, w Written) // Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil follows the controller's alone. Grants agent.Watcher // Fetch reads an object of the grants part's bucket by its name; Open opens a box sealed to this node's key; Key is // this node's sealing key's public half (SPKI PEM), said in the have so the grants part seals to it ("" unknown). Fetch func(ctx context.Context, name string) ([]byte, error) Open func(seal.Box) ([]byte, error) Key func() string // Kept is the grants part's key as last heard on this machine and its revision (nil when none), and Keep keeps it: // what the follower follows while the key cannot be watched (ADR 0547 clause 8). Backoff is the wait before the // try-th watch again. Kept func() ([]byte, uint64) Keep func(value []byte, rev uint64) Backoff func(try int) time.Duration // VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the // key the loaded list's hashes are handed sealed to. Nil hands nothing. VaultKey func() string Now func() time.Time Say func(string) } type source struct { key string module string rev uint64 resource json.RawMessage wanted string } // Follower follows the node's compositions for the bus's user list. type Follower struct { env Env kick chan struct{} mu sync.Mutex srcs map[string]source // by composition key // cur is the source last taken, and written whether its write went through. cur source written bool state Written mod string said map[string]bool // grants is the grants part's key as last heard (nil when none), and following the source last taken: FollowGrants // or FollowController. grants *delivery.GrantsUserList grantsRev uint64 following string // unwatched is why the grants part's key is not watched now; empty while it is, or where it need not be. unwatched string } // New is a follower for a node. func New(env Env) *Follower { if env.Now == nil { env.Now = time.Now } if env.Say == nil { env.Say = func(string) {} } if env.Backoff == nil { env.Backoff = func(int) time.Duration { return time.Minute } } f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}} if env.Grants != nil && env.Kept != nil { if raw, rev := env.Kept(); raw != nil { if g, err := delivery.DecodeGrantsUserList(raw); err == nil { f.grants, f.grantsRev = &g, rev } } } return f } // Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module, // and when a person asks. It never blocks. func (f *Follower) Again() { select { case f.kick <- struct{}{}: default: } } // State is the module whose list the follower writes ("" when no composition of this node carries one) and what it // last did. func (f *Follower) State() (string, Written) { f.mu.Lock() defer f.mu.Unlock() return f.mod, f.state } // Run follows the compositions until ctx ends. func (f *Follower) Run(ctx context.Context) error { updates, err := f.env.Composed.Watch(ctx, f.env.Node+".>") if err != nil { return fmt.Errorf("the user list cannot be followed: %w", err) } caught := false // The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk // buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link) // never writes the controller's list after take-over before it heard the switch; a watch refused or ended is tried // again on the backoff, and meanwhile the follow last heard (kept on the machine) holds. var grants <-chan agent.Update var retry <-chan time.Time opened, grantsCaught, tries := false, false, 0 failed := func(why string) { grants, tries = nil, tries+1 retry = time.After(f.env.Backoff(tries)) f.mu.Lock() f.unwatched = why f.mu.Unlock() f.sayOnce("the grants part's user-list key is not watched, tried again: " + why) } open := func() { if grants != nil || retry != nil || f.env.Grants == nil || !f.busNode() { return } opened = true ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey) if err != nil { failed(err.Error()) return } grants = ch } for { again := false select { case <-ctx.Done(): return nil case <-f.kick: again = true case <-retry: retry = nil case u, ok := <-updates: if !ok { return fmt.Errorf("the watch of the compositions ended") } if u.Caught { caught = true } else { f.take(u) } case u, ok := <-grants: if !ok { failed("its watch ended") break } if u.Caught { grantsCaught, tries = true, 0 f.mu.Lock() f.unwatched = "" f.mu.Unlock() break } f.takeGrants(u) } if !caught { continue } open() if opened && !grantsCaught && grants != nil { continue // the switch is not heard yet: nothing is written before it is } f.mu.Lock() if f.unwatched != f.state.Unwatched { again = true // the have says whether the key is watched } f.mu.Unlock() f.settle(ctx, again) } } // busNode says some composition of this node carries the bus's user list. func (f *Follower) busNode() bool { f.mu.Lock() defer f.mu.Unlock() return f.cur.key != "" || len(f.srcs) > 0 } // takeGrants keeps the grants part's key as heard; a value it cannot read is said and leaves the controller's list // followed (no list is followed that the node cannot check). func (f *Follower) takeGrants(u agent.Update) { f.mu.Lock() defer f.mu.Unlock() f.grantsRev = u.Revision if u.Deleted { f.grants = nil f.keep(nil, 0) return } g, err := delivery.DecodeGrantsUserList(u.Value) if err != nil { f.grants = nil f.keep(nil, 0) f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error()) return } f.grants = &g f.keep(u.Value, u.Revision) } // keep keeps the key last heard on the machine (Env.Keep), so a follower that cannot watch it follows what it said. func (f *Follower) keep(value []byte, rev uint64) { if f.env.Keep != nil { f.env.Keep(value, rev) } } // Following is which list the follower takes now: the grants part's while its key says so, the controller's otherwise. func (f *Follower) Following() string { f.mu.Lock() defer f.mu.Unlock() return f.followingLocked() } func (f *Follower) followingLocked() string { if f.grants != nil && f.grants.Follow == delivery.FollowGrants { return delivery.FollowGrants } return delivery.FollowController } func (f *Follower) unwatchedNow() string { f.mu.Lock() defer f.mu.Unlock() return f.unwatched } func (f *Follower) key() string { if f.env.Key == nil { return "" } return f.env.Key() } // take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted. func (f *Follower) take(u agent.Update) { node, module, _, isNode, err := delivery.ParseComposedKey(u.Key) if err != nil || node != f.env.Node || isNode { return } f.mu.Lock() defer f.mu.Unlock() if u.Deleted { delete(f.srcs, u.Key) return } c, err := delivery.DecodeComposed(u.Value) if err != nil { delete(f.srcs, u.Key) return } var trusted []ed25519.PublicKey if f.env.Trusted != nil { trusted = f.env.Trusted() } if _, err := delivery.VerifyComposed(c, trusted); err != nil { // The agent says it is refused; a refused composition is never a source of the list. delete(f.srcs, u.Key) return } if c.Node != f.env.Node || c.Module != module { // Signed, and for another node or module than its key says: a re-put of another composition is no source. delete(f.srcs, u.Key) f.sayOnceLocked(fmt.Sprintf("the composition under %s is %s's of %s, and is no source of the bus's user list", u.Key, c.Node, c.Module)) return } raw, content, digest, ok := userList(c.Resources) if !ok { delete(f.srcs, u.Key) return } wanted := digest if wanted == "" { wanted = Digest([]byte(content)) } f.srcs[u.Key] = source{key: u.Key, module: module, rev: u.Revision, resource: raw, wanted: wanted} } // settle writes the newest source's list when it is newer than the one written, or the one written again when asked. // A source never goes back: a key deleted, or a newer one refused, leaves the list written as it is, since an older // list would take away accounts a newer one gave. func (f *Follower) settle(ctx context.Context, again bool) { f.mu.Lock() follow := f.followingLocked() if follow != f.following { // The switch turned: the other source is written, though neither changed. again = true } f.mu.Unlock() if follow == delivery.FollowGrants { f.settleGrants(ctx, again) return } f.mu.Lock() newest := f.cur for _, s := range f.srcs { if s.rev > newest.rev { newest = s } } if newest.key == "" || (newest.rev == f.cur.rev && f.written && !again) { f.mu.Unlock() return } f.cur = newest at := fmt.Sprintf("%s@%d", newest.key, newest.rev) f.mu.Unlock() w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(), Follows: delivery.FollowController, Key: f.key(), Unwatched: f.unwatchedNow()} var changed bool body, err := f.env.Resolve(ctx, newest.resource) if err == nil { w.Users = Users(body) err = f.guard(newest.resource, body) if err != nil { // Not written: what is on the machine stays, and the have says why, loudly. if disk, derr := f.env.OnDisk(newest.resource); derr == nil && disk != nil { w.Digest = Digest(disk) } } } var loaded []byte if err == nil { w.Digest, changed, err = f.env.Write(ctx, newest.module, newest.resource, body) if err == nil { loaded = body } } else if body != nil && f.env.OnDisk != nil { // Refused: the list on the machine stays, and it is the one the bus has loaded. loaded, _ = f.env.OnDisk(newest.resource) } if err != nil { w.Error = err.Error() } f.mu.Lock() prev := f.state f.mu.Unlock() w.Handed = f.hand(loaded, prev.Handed) f.mu.Lock() f.mod, f.state = newest.module, w f.written = err == nil f.following = delivery.FollowController f.mu.Unlock() if err != nil { f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err)) } else if changed { f.env.Say(fmt.Sprintf("the bus's user list from %s is written alone (%d users, %s); the bus reloads it itself", at, w.Users, short(w.Digest))) } // Reported when what it says changed: a look again that found the file as it was says nothing new. if f.env.Report != nil && (changed || reportable(prev, w)) { f.env.Report(ctx, newest.module, w) } } // reportable says a have's user list says something new: a look again that found the file as it was says nothing. func reportable(prev, w Written) bool { return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error || prev.Follows != w.Follows || prev.Key != w.Key || handedKey(prev.Handed) != handedKey(w.Handed) || prev.Unwatched != w.Unwatched } // settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of // the bus module declares, made 0600 (ADR 0547 clause 7): when the key moved, when the switch turned, or when asked. func (f *Follower) settleGrants(ctx context.Context, again bool) { f.mu.Lock() g, rev := *f.grants, f.grantsRev file := f.cur for _, s := range f.srcs { if s.rev > file.rev { file = s } } if !again && f.following == delivery.FollowGrants && f.state.Revision == rev && f.written { f.mu.Unlock() return } f.mu.Unlock() w := Written{Source: delivery.GrantsUserListSource, Revision: rev, Wanted: g.Digest, At: f.env.Now().UTC(), Follows: delivery.FollowGrants, Key: f.key(), Unwatched: f.unwatchedNow()} var err error var changed bool resource, rerr := mode0600(file.resource) switch { case file.key == "" || rerr != nil: err = fmt.Errorf("no composition of this node declares the bus's user-list file, so the grants part's list has nowhere to be written") case g.Digest == "": err = fmt.Errorf("the grants part's key names no list yet; the list on the machine is kept") case f.env.Fetch == nil || f.env.Open == nil: err = fmt.Errorf("this engine cannot read or open the grants part's list") case g.SealedTo != "" && w.Key != "" && g.SealedTo != delivery.SealingKeyID(w.Key): err = fmt.Errorf("the grants part's list is sealed to another key than this node's (%s); the list on the machine is kept until it seals to this one", g.SealedTo) } var body []byte if err == nil { var object []byte if object, err = f.env.Fetch(ctx, g.Object); err == nil { body, err = delivery.OpenUserList(object, g.Digest, f.env.Open) } if err != nil { err = fmt.Errorf("the grants part's list %s could not be had: %w", short(g.Digest), err) } } if err == nil { w.Users = Users(body) if err = f.guard(resource, body); err != nil { if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil { w.Digest = Digest(disk) } } } if err == nil { w.Digest, changed, err = f.env.Write(ctx, file.module, resource, body) } else if w.Digest == "" && file.key != "" && f.env.OnDisk != nil { if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil { w.Digest = Digest(disk) } } if err != nil { w.Error = err.Error() } f.mu.Lock() prev := f.state // The grants part's list is not handed to the vault: the handing is of what the controller's list loaded (clause 9). w.Handed = prev.Handed f.mod, f.state = file.module, w f.written = err == nil f.following = delivery.FollowGrants f.mu.Unlock() at := fmt.Sprintf("%s@%d", delivery.GrantsUserListSource, rev) if err != nil { f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err)) } else if changed { f.env.Say(fmt.Sprintf("the bus's user list from the grants part (%s, revision %d) is written alone (%d users, %s); the bus reloads it itself", at, g.Revision, w.Users, short(w.Digest))) } if f.env.Report != nil && file.module != "" && (changed || reportable(prev, w)) { f.env.Report(ctx, file.module, w) } } // Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus // module's composition from the controller carries the controller's list, so its user-list resource holds the list on the // machine instead (made 0600), or is left out when the machine holds none. A list on the machine that cannot be read // is an error: left out, the apply would remove the file as an orphan and the bus would refuse every connection at its // next reload, so the bus module's apply waits instead. Any other composition, or the controller's list followed, is c // unchanged. func (f *Follower) Hold(c delivery.Composed) (delivery.Composed, error) { if f.Following() != delivery.FollowGrants { return c, nil } out := c out.Resources = nil for _, raw := range c.Resources { if r, _, _, ok := userList([]json.RawMessage{raw}); ok { var held json.RawMessage body, err := f.env.OnDisk(r) if err != nil { return c, fmt.Errorf("the user list on the machine could not be read, so the apply waits and the file stays: %w", err) } if body != nil { var m map[string]any if json.Unmarshal(r, &m) == nil { delete(m, "digest") delete(m, "size") m["content"], m["mode"] = string(body), "0600" held, _ = json.Marshal(m) } } if held != nil { out.Resources = append(out.Resources, held) } continue } out.Resources = append(out.Resources, raw) } return out, nil } // mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7). func mode0600(raw json.RawMessage) (json.RawMessage, error) { if len(raw) == 0 { return nil, fmt.Errorf("no resource") } var r map[string]any if err := json.Unmarshal(raw, &r); err != nil { return nil, err } r["mode"] = "0600" return json.Marshal(r) } // hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a // new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash. func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded { if f.env.VaultKey == nil || loaded == nil { return before } key := f.env.VaultKey() if key == "" { return before } from := Digest(loaded) if before != nil && before.From == from && before.To == delivery.KeyFingerprint(key) { return before } accounts := delivery.HandedAccountsOf(loaded) if len(accounts) == 0 { return nil } h, err := delivery.HandAccounts(accounts, key, from, f.env.Now()) if err != nil { f.sayOnce(fmt.Sprintf("the loaded list's hashes are not handed to the vault: %v", err)) return before } f.env.Say(fmt.Sprintf("the hashes of the %d users of the bus's loaded list (%s) are handed to the vault, sealed to "+ "its key %s, in the bus module's have", h.Users, short(from), short(h.To))) return &h } // handedKey is what a handing says that a report compares: its list and key. func handedKey(h *delivery.AccountsHanded) string { if h == nil { return "" } return h.From + " " + h.To } // ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus. const ControllerUser = "controller" // guard refuses a list that would lock the mesh out of its own bus: one without the controller's own user, or one that // drops more than half the users the file on the machine holds. A list that shrinks that far is a composer that read // half the mesh, not half the mesh gone; it waits for a person, and the bus keeps the list it has. func (f *Follower) guard(resource json.RawMessage, body []byte) error { names := Names(body) if !slices.Contains(names, ControllerUser) { return fmt.Errorf("refused: the list holds no user %q, and written it would lock the controller out of the "+ "bus; the list on the machine is kept", ControllerUser) } if f.env.OnDisk == nil { return nil } disk, err := f.env.OnDisk(resource) if err != nil || disk == nil { return nil } if had := Users(disk); had > 0 && len(names)*2 < had { return fmt.Errorf("refused: the list holds %d users and the one on the machine %d; a list that drops more "+ "than half is not written, and the list on the machine is kept", len(names), had) } return nil } func (f *Follower) sayOnceLocked(line string) { if !f.said[line] { f.said[line] = true f.env.Say(line) } } func (f *Follower) sayOnce(line string) { f.mu.Lock() said := f.said[line] f.said[line] = true f.mu.Unlock() if !said { f.env.Say(line) } } // userList is the bus's user list among a composition's resources: the resource, its content, and its digest when it // is given by digest rather than content. func userList(resources []json.RawMessage) (json.RawMessage, string, string, bool) { for _, raw := range resources { var head struct { ID string `json:"id"` Type string `json:"type"` Content string `json:"content"` Digest string `json:"digest"` } if json.Unmarshal(raw, &head) != nil || head.Type != "file" { continue } if head.ID == ResourceID || strings.HasSuffix(head.ID, "."+ResourceID) { return raw, head.Content, head.Digest, true } } return nil, "", "", false } // Digest is content's digest as the have says it, `sha256:`. func Digest(content []byte) string { sum := sha256.Sum256(content) return "sha256:" + hex.EncodeToString(sum[:]) } // userEntry is one user of the list as the composer writes it (and the bus module's own tools read it). var userEntry = regexp.MustCompile(`\{\s*user:\s*"([^"]+)"`) // Users is how many users a list holds. func Users(content []byte) int { return len(userEntry.FindAllIndex(content, -1)) } // Names is the users a list holds, as it names them. func Names(content []byte) []string { var out []string for _, m := range userEntry.FindAllSubmatch(content, -1) { out = append(out, string(m[1])) } return out } func short(d string) string { d = strings.TrimPrefix(d, "sha256:") if len(d) > 12 { d = d[:12] } return d }