Files
mesh-host/internal/userlist/userlist.go
T
jschoubben e88ddab241
mesh/delivery merged: source.main moved to novox/mesh-host@7f04d1204765
mesh/delivery-group group feat/1945-user-list-switch merged: every member is merged
mesh/check being carried to e88ddab24170 on 7f04d1204765 | novox/mesh-host#145 in group feat/1945-user-list-switch
An apply of the bus module waits when the list on the machine cannot be read while the grants part's is followed, so the file is never removed as an orphan (review of the switch, issue 1945)
2026-10-26 12:54:06 +01:00

696 lines
24 KiB
Go

// Package userlist is the bridge of novox/hq ADR 0547 §8: the node agent of the node holding mesh-broker applies the
// bus's user list from the newest composition of the bus module the controller writes for its node, alone and with no
// want, so a new account is in force on the bus without anybody's hand.
//
// **Why it exists.** The list reached the bus only through a want of the bus module, and ADR 0539 §15 bars the assigner
// from writing one: on 2026-10-25 it stood from 11:23 to 20:50 and every new account was refused until the operator put
// it back by hand (issue 1945). The bus module's entrypoint already reloads the server in place when the file changes;
// what was missing is a writer of the file that does not wait on a want.
//
// **It watches, it never polls.** It follows the node's composition bucket. Each composition is verified as the agent
// verifies it (signed by the controller's or the operator's key) before anything of it is read. Of those that carry the
// bus's user list (a file resource `bus-users`), the newest by revision is the source; its list is written alone
// (apply.WriteAlone) when it differs from the file on the machine, and what was written is reported in the bus
// module's have. It looks again after every apply of the bus module, since an apply of an older composition writes an
// older list, and when asked (Again).
//
// It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list.
//
// **It hands the loaded list's hashes to the vault** (ADR 0547 clause 9, issue 1945): once the vault's key is known
// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to
// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only
// when the list on the machine or the vault's key changes.
//
// **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose
// compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says
// FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked
// against the key's digest, and written to the same file (mode 0600) only when the file's digest differs; otherwise the
// controller's compositions are the source, as before. Never both. The have says which it follows and this node's key,
// which the grants part seals to.
package userlist
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"regexp"
"slices"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/seal"
"github.com/novox/mesh-host/internal/agent"
)
// ResourceID is the id the composer gives the bus's user list (mesh-sdk catalogue.BusUsersID); in a composition it may
// carry its module before it, `<module>.bus-users`.
const ResourceID = "bus-users"
// Written is what the follower last did: the source it took the list from, the list's digest there, the file's digest
// on the machine after the write, how many users the list holds, when, and why a write failed.
type Written = delivery.UserListWritten
// Env is everything the follower needs of the node.
type Env struct {
Node string
Composed agent.Watcher
// Trusted is the keys a composition may be signed by, as the agent trusts them.
Trusted func() []ed25519.PublicKey
// Resolve is the list a user-list resource holds: its content, or the object its digest names, fetched and
// verified (the controller sends a file over 64 KiB by digest, ADR 0486 §1, and the live list is over it).
Resolve func(ctx context.Context, resource json.RawMessage) ([]byte, error)
// OnDisk is what the resource's file holds on the machine now; nil when there is none.
OnDisk func(resource json.RawMessage) ([]byte, error)
// Write writes the user list's file resource of a module's composition alone, with body as its content, when it
// differs from the file on the machine, and answers the digest of the file there after it (`sha256:<hex>`) and
// whether it changed.
Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error)
// Report says what was written, in the module's have.
Report func(ctx context.Context, module string, w Written)
// Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil follows the controller's alone.
Grants agent.Watcher
// Fetch reads an object of the grants part's bucket by its name; Open opens a box sealed to this node's key; Key is
// this node's sealing key's public half (SPKI PEM), said in the have so the grants part seals to it ("" unknown).
Fetch func(ctx context.Context, name string) ([]byte, error)
Open func(seal.Box) ([]byte, error)
Key func() string
// Kept is the grants part's key as last heard on this machine and its revision (nil when none), and Keep keeps it:
// what the follower follows while the key cannot be watched (ADR 0547 clause 8). Backoff is the wait before the
// try-th watch again.
Kept func() ([]byte, uint64)
Keep func(value []byte, rev uint64)
Backoff func(try int) time.Duration
// VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the
// key the loaded list's hashes are handed sealed to. Nil hands nothing.
VaultKey func() string
Now func() time.Time
Say func(string)
}
type source struct {
key string
module string
rev uint64
resource json.RawMessage
wanted string
}
// Follower follows the node's compositions for the bus's user list.
type Follower struct {
env Env
kick chan struct{}
mu sync.Mutex
srcs map[string]source // by composition key
// cur is the source last taken, and written whether its write went through.
cur source
written bool
state Written
mod string
said map[string]bool
// grants is the grants part's key as last heard (nil when none), and following the source last taken: FollowGrants
// or FollowController.
grants *delivery.GrantsUserList
grantsRev uint64
following string
// unwatched is why the grants part's key is not watched now; empty while it is, or where it need not be.
unwatched string
}
// New is a follower for a node.
func New(env Env) *Follower {
if env.Now == nil {
env.Now = time.Now
}
if env.Say == nil {
env.Say = func(string) {}
}
if env.Backoff == nil {
env.Backoff = func(int) time.Duration { return time.Minute }
}
f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
if env.Grants != nil && env.Kept != nil {
if raw, rev := env.Kept(); raw != nil {
if g, err := delivery.DecodeGrantsUserList(raw); err == nil {
f.grants, f.grantsRev = &g, rev
}
}
}
return f
}
// Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module,
// and when a person asks. It never blocks.
func (f *Follower) Again() {
select {
case f.kick <- struct{}{}:
default:
}
}
// State is the module whose list the follower writes ("" when no composition of this node carries one) and what it
// last did.
func (f *Follower) State() (string, Written) {
f.mu.Lock()
defer f.mu.Unlock()
return f.mod, f.state
}
// Run follows the compositions until ctx ends.
func (f *Follower) Run(ctx context.Context) error {
updates, err := f.env.Composed.Watch(ctx, f.env.Node+".>")
if err != nil {
return fmt.Errorf("the user list cannot be followed: %w", err)
}
caught := false
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
// never writes the controller's list after take-over before it heard the switch; a watch refused or ended is tried
// again on the backoff, and meanwhile the follow last heard (kept on the machine) holds.
var grants <-chan agent.Update
var retry <-chan time.Time
opened, grantsCaught, tries := false, false, 0
failed := func(why string) {
grants, tries = nil, tries+1
retry = time.After(f.env.Backoff(tries))
f.mu.Lock()
f.unwatched = why
f.mu.Unlock()
f.sayOnce("the grants part's user-list key is not watched, tried again: " + why)
}
open := func() {
if grants != nil || retry != nil || f.env.Grants == nil || !f.busNode() {
return
}
opened = true
ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey)
if err != nil {
failed(err.Error())
return
}
grants = ch
}
for {
again := false
select {
case <-ctx.Done():
return nil
case <-f.kick:
again = true
case <-retry:
retry = nil
case u, ok := <-updates:
if !ok {
return fmt.Errorf("the watch of the compositions ended")
}
if u.Caught {
caught = true
} else {
f.take(u)
}
case u, ok := <-grants:
if !ok {
failed("its watch ended")
break
}
if u.Caught {
grantsCaught, tries = true, 0
f.mu.Lock()
f.unwatched = ""
f.mu.Unlock()
break
}
f.takeGrants(u)
}
if !caught {
continue
}
open()
if opened && !grantsCaught && grants != nil {
continue // the switch is not heard yet: nothing is written before it is
}
f.mu.Lock()
if f.unwatched != f.state.Unwatched {
again = true // the have says whether the key is watched
}
f.mu.Unlock()
f.settle(ctx, again)
}
}
// busNode says some composition of this node carries the bus's user list.
func (f *Follower) busNode() bool {
f.mu.Lock()
defer f.mu.Unlock()
return f.cur.key != "" || len(f.srcs) > 0
}
// takeGrants keeps the grants part's key as heard; a value it cannot read is said and leaves the controller's list
// followed (no list is followed that the node cannot check).
func (f *Follower) takeGrants(u agent.Update) {
f.mu.Lock()
defer f.mu.Unlock()
f.grantsRev = u.Revision
if u.Deleted {
f.grants = nil
f.keep(nil, 0)
return
}
g, err := delivery.DecodeGrantsUserList(u.Value)
if err != nil {
f.grants = nil
f.keep(nil, 0)
f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error())
return
}
f.grants = &g
f.keep(u.Value, u.Revision)
}
// keep keeps the key last heard on the machine (Env.Keep), so a follower that cannot watch it follows what it said.
func (f *Follower) keep(value []byte, rev uint64) {
if f.env.Keep != nil {
f.env.Keep(value, rev)
}
}
// Following is which list the follower takes now: the grants part's while its key says so, the controller's otherwise.
func (f *Follower) Following() string {
f.mu.Lock()
defer f.mu.Unlock()
return f.followingLocked()
}
func (f *Follower) followingLocked() string {
if f.grants != nil && f.grants.Follow == delivery.FollowGrants {
return delivery.FollowGrants
}
return delivery.FollowController
}
func (f *Follower) unwatchedNow() string {
f.mu.Lock()
defer f.mu.Unlock()
return f.unwatched
}
func (f *Follower) key() string {
if f.env.Key == nil {
return ""
}
return f.env.Key()
}
// take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted.
func (f *Follower) take(u agent.Update) {
node, module, _, isNode, err := delivery.ParseComposedKey(u.Key)
if err != nil || node != f.env.Node || isNode {
return
}
f.mu.Lock()
defer f.mu.Unlock()
if u.Deleted {
delete(f.srcs, u.Key)
return
}
c, err := delivery.DecodeComposed(u.Value)
if err != nil {
delete(f.srcs, u.Key)
return
}
var trusted []ed25519.PublicKey
if f.env.Trusted != nil {
trusted = f.env.Trusted()
}
if _, err := delivery.VerifyComposed(c, trusted); err != nil {
// The agent says it is refused; a refused composition is never a source of the list.
delete(f.srcs, u.Key)
return
}
if c.Node != f.env.Node || c.Module != module {
// Signed, and for another node or module than its key says: a re-put of another composition is no source.
delete(f.srcs, u.Key)
f.sayOnceLocked(fmt.Sprintf("the composition under %s is %s's of %s, and is no source of the bus's user list",
u.Key, c.Node, c.Module))
return
}
raw, content, digest, ok := userList(c.Resources)
if !ok {
delete(f.srcs, u.Key)
return
}
wanted := digest
if wanted == "" {
wanted = Digest([]byte(content))
}
f.srcs[u.Key] = source{key: u.Key, module: module, rev: u.Revision, resource: raw, wanted: wanted}
}
// settle writes the newest source's list when it is newer than the one written, or the one written again when asked.
// A source never goes back: a key deleted, or a newer one refused, leaves the list written as it is, since an older
// list would take away accounts a newer one gave.
func (f *Follower) settle(ctx context.Context, again bool) {
f.mu.Lock()
follow := f.followingLocked()
if follow != f.following {
// The switch turned: the other source is written, though neither changed.
again = true
}
f.mu.Unlock()
if follow == delivery.FollowGrants {
f.settleGrants(ctx, again)
return
}
f.mu.Lock()
newest := f.cur
for _, s := range f.srcs {
if s.rev > newest.rev {
newest = s
}
}
if newest.key == "" || (newest.rev == f.cur.rev && f.written && !again) {
f.mu.Unlock()
return
}
f.cur = newest
at := fmt.Sprintf("%s@%d", newest.key, newest.rev)
f.mu.Unlock()
w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(),
Follows: delivery.FollowController, Key: f.key(), Unwatched: f.unwatchedNow()}
var changed bool
body, err := f.env.Resolve(ctx, newest.resource)
if err == nil {
w.Users = Users(body)
err = f.guard(newest.resource, body)
if err != nil {
// Not written: what is on the machine stays, and the have says why, loudly.
if disk, derr := f.env.OnDisk(newest.resource); derr == nil && disk != nil {
w.Digest = Digest(disk)
}
}
}
var loaded []byte
if err == nil {
w.Digest, changed, err = f.env.Write(ctx, newest.module, newest.resource, body)
if err == nil {
loaded = body
}
} else if body != nil && f.env.OnDisk != nil {
// Refused: the list on the machine stays, and it is the one the bus has loaded.
loaded, _ = f.env.OnDisk(newest.resource)
}
if err != nil {
w.Error = err.Error()
}
f.mu.Lock()
prev := f.state
f.mu.Unlock()
w.Handed = f.hand(loaded, prev.Handed)
f.mu.Lock()
f.mod, f.state = newest.module, w
f.written = err == nil
f.following = delivery.FollowController
f.mu.Unlock()
if err != nil {
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
} else if changed {
f.env.Say(fmt.Sprintf("the bus's user list from %s is written alone (%d users, %s); the bus reloads it itself",
at, w.Users, short(w.Digest)))
}
// Reported when what it says changed: a look again that found the file as it was says nothing new.
if f.env.Report != nil && (changed || reportable(prev, w)) {
f.env.Report(ctx, newest.module, w)
}
}
// reportable says a have's user list says something new: a look again that found the file as it was says nothing.
func reportable(prev, w Written) bool {
return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error ||
prev.Follows != w.Follows || prev.Key != w.Key || handedKey(prev.Handed) != handedKey(w.Handed) ||
prev.Unwatched != w.Unwatched
}
// settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of
// the bus module declares, made 0600 (ADR 0547 clause 7): when the key moved, when the switch turned, or when asked.
func (f *Follower) settleGrants(ctx context.Context, again bool) {
f.mu.Lock()
g, rev := *f.grants, f.grantsRev
file := f.cur
for _, s := range f.srcs {
if s.rev > file.rev {
file = s
}
}
if !again && f.following == delivery.FollowGrants && f.state.Revision == rev && f.written {
f.mu.Unlock()
return
}
f.mu.Unlock()
w := Written{Source: delivery.GrantsUserListSource, Revision: rev, Wanted: g.Digest, At: f.env.Now().UTC(),
Follows: delivery.FollowGrants, Key: f.key(), Unwatched: f.unwatchedNow()}
var err error
var changed bool
resource, rerr := mode0600(file.resource)
switch {
case file.key == "" || rerr != nil:
err = fmt.Errorf("no composition of this node declares the bus's user-list file, so the grants part's list has nowhere to be written")
case g.Digest == "":
err = fmt.Errorf("the grants part's key names no list yet; the list on the machine is kept")
case f.env.Fetch == nil || f.env.Open == nil:
err = fmt.Errorf("this engine cannot read or open the grants part's list")
case g.SealedTo != "" && w.Key != "" && g.SealedTo != delivery.SealingKeyID(w.Key):
err = fmt.Errorf("the grants part's list is sealed to another key than this node's (%s); the list on the machine is kept until it seals to this one", g.SealedTo)
}
var body []byte
if err == nil {
var object []byte
if object, err = f.env.Fetch(ctx, g.Object); err == nil {
body, err = delivery.OpenUserList(object, g.Digest, f.env.Open)
}
if err != nil {
err = fmt.Errorf("the grants part's list %s could not be had: %w", short(g.Digest), err)
}
}
if err == nil {
w.Users = Users(body)
if err = f.guard(resource, body); err != nil {
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
w.Digest = Digest(disk)
}
}
}
if err == nil {
w.Digest, changed, err = f.env.Write(ctx, file.module, resource, body)
} else if w.Digest == "" && file.key != "" && f.env.OnDisk != nil {
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
w.Digest = Digest(disk)
}
}
if err != nil {
w.Error = err.Error()
}
f.mu.Lock()
prev := f.state
// The grants part's list is not handed to the vault: the handing is of what the controller's list loaded (clause 9).
w.Handed = prev.Handed
f.mod, f.state = file.module, w
f.written = err == nil
f.following = delivery.FollowGrants
f.mu.Unlock()
at := fmt.Sprintf("%s@%d", delivery.GrantsUserListSource, rev)
if err != nil {
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
} else if changed {
f.env.Say(fmt.Sprintf("the bus's user list from the grants part (%s, revision %d) is written alone (%d users, %s); the bus reloads it itself",
at, g.Revision, w.Users, short(w.Digest)))
}
if f.env.Report != nil && file.module != "" && (changed || reportable(prev, w)) {
f.env.Report(ctx, file.module, w)
}
}
// Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus
// module's composition from the controller carries the controller's list, so its user-list resource holds the list on the
// machine instead (made 0600), or is left out when the machine holds none. A list on the machine that cannot be read
// is an error: left out, the apply would remove the file as an orphan and the bus would refuse every connection at its
// next reload, so the bus module's apply waits instead. Any other composition, or the controller's list followed, is c
// unchanged.
func (f *Follower) Hold(c delivery.Composed) (delivery.Composed, error) {
if f.Following() != delivery.FollowGrants {
return c, nil
}
out := c
out.Resources = nil
for _, raw := range c.Resources {
if r, _, _, ok := userList([]json.RawMessage{raw}); ok {
var held json.RawMessage
body, err := f.env.OnDisk(r)
if err != nil {
return c, fmt.Errorf("the user list on the machine could not be read, so the apply waits and the file stays: %w", err)
}
if body != nil {
var m map[string]any
if json.Unmarshal(r, &m) == nil {
delete(m, "digest")
delete(m, "size")
m["content"], m["mode"] = string(body), "0600"
held, _ = json.Marshal(m)
}
}
if held != nil {
out.Resources = append(out.Resources, held)
}
continue
}
out.Resources = append(out.Resources, raw)
}
return out, nil
}
// mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7).
func mode0600(raw json.RawMessage) (json.RawMessage, error) {
if len(raw) == 0 {
return nil, fmt.Errorf("no resource")
}
var r map[string]any
if err := json.Unmarshal(raw, &r); err != nil {
return nil, err
}
r["mode"] = "0600"
return json.Marshal(r)
}
// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a
// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash.
func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded {
if f.env.VaultKey == nil || loaded == nil {
return before
}
key := f.env.VaultKey()
if key == "" {
return before
}
from := Digest(loaded)
if before != nil && before.From == from && before.To == delivery.KeyFingerprint(key) {
return before
}
accounts := delivery.HandedAccountsOf(loaded)
if len(accounts) == 0 {
return nil
}
h, err := delivery.HandAccounts(accounts, key, from, f.env.Now())
if err != nil {
f.sayOnce(fmt.Sprintf("the loaded list's hashes are not handed to the vault: %v", err))
return before
}
f.env.Say(fmt.Sprintf("the hashes of the %d users of the bus's loaded list (%s) are handed to the vault, sealed to "+
"its key %s, in the bus module's have", h.Users, short(from), short(h.To)))
return &h
}
// handedKey is what a handing says that a report compares: its list and key.
func handedKey(h *delivery.AccountsHanded) string {
if h == nil {
return ""
}
return h.From + " " + h.To
}
// ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus.
const ControllerUser = "controller"
// guard refuses a list that would lock the mesh out of its own bus: one without the controller's own user, or one that
// drops more than half the users the file on the machine holds. A list that shrinks that far is a composer that read
// half the mesh, not half the mesh gone; it waits for a person, and the bus keeps the list it has.
func (f *Follower) guard(resource json.RawMessage, body []byte) error {
names := Names(body)
if !slices.Contains(names, ControllerUser) {
return fmt.Errorf("refused: the list holds no user %q, and written it would lock the controller out of the "+
"bus; the list on the machine is kept", ControllerUser)
}
if f.env.OnDisk == nil {
return nil
}
disk, err := f.env.OnDisk(resource)
if err != nil || disk == nil {
return nil
}
if had := Users(disk); had > 0 && len(names)*2 < had {
return fmt.Errorf("refused: the list holds %d users and the one on the machine %d; a list that drops more "+
"than half is not written, and the list on the machine is kept", len(names), had)
}
return nil
}
func (f *Follower) sayOnceLocked(line string) {
if !f.said[line] {
f.said[line] = true
f.env.Say(line)
}
}
func (f *Follower) sayOnce(line string) {
f.mu.Lock()
said := f.said[line]
f.said[line] = true
f.mu.Unlock()
if !said {
f.env.Say(line)
}
}
// userList is the bus's user list among a composition's resources: the resource, its content, and its digest when it
// is given by digest rather than content.
func userList(resources []json.RawMessage) (json.RawMessage, string, string, bool) {
for _, raw := range resources {
var head struct {
ID string `json:"id"`
Type string `json:"type"`
Content string `json:"content"`
Digest string `json:"digest"`
}
if json.Unmarshal(raw, &head) != nil || head.Type != "file" {
continue
}
if head.ID == ResourceID || strings.HasSuffix(head.ID, "."+ResourceID) {
return raw, head.Content, head.Digest, true
}
}
return nil, "", "", false
}
// Digest is content's digest as the have says it, `sha256:<hex>`.
func Digest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
// userEntry is one user of the list as the composer writes it (and the bus module's own tools read it).
var userEntry = regexp.MustCompile(`\{\s*user:\s*"([^"]+)"`)
// Users is how many users a list holds.
func Users(content []byte) int { return len(userEntry.FindAllIndex(content, -1)) }
// Names is the users a list holds, as it names them.
func Names(content []byte) []string {
var out []string
for _, m := range userEntry.FindAllSubmatch(content, -1) {
out = append(out, string(m[1]))
}
return out
}
func short(d string) string {
d = strings.TrimPrefix(d, "sha256:")
if len(d) > 12 {
d = d[:12]
}
return d
}