From 5d5ee69775854b34f4ae44ee0b332aeec9b6908a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:17:28 +0100 Subject: [PATCH 01/20] Add the module mesh-enrolment: the seat's skeleton, its bucket of open enrolments and the tokens verb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The holder of the seat mesh-enrolment (ADR 0587 §1, §2, §8) runs as a bundle on its own account, keeps the bucket enrolments with a time to live per key, and answers tokens: every open enrolment or one node's with stage, expiry, ask and revision, the bucket's fill, whether issuing is on and why, and for one node at live the token sealed by the vault and relayed, kept nowhere. Tests added: two (one table-style test of tokens, the manifest test); the repository is over its budget, paid by merging tests in the assigner's seat tests in 774/S59's commit. (issue 774, 774/S54) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/enrolment.go | 144 ++++++++++ .../cmd/mesh-enrolment/holder.go | 263 ++++++++++++++++++ .../cmd/mesh-enrolment/kit_test.go | 240 ++++++++++++++++ .../mesh-enrolment/cmd/mesh-enrolment/main.go | 146 ++++++++++ .../cmd/mesh-enrolment/tokens.go | 118 ++++++++ .../cmd/mesh-enrolment/tokens_test.go | 106 +++++++ .../cmd/mesh-enrolment/tools.go | 51 ++++ .../cmd/mesh-enrolment/vault.go | 70 +++++ modules/mesh-enrolment/go.mod | 16 ++ modules/mesh-enrolment/go.sum | 26 ++ modules/mesh-enrolment/module.json | 88 ++++++ 11 files changed, 1268 insertions(+) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/holder.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/main.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/tools.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/vault.go create mode 100644 modules/mesh-enrolment/go.mod create mode 100644 modules/mesh-enrolment/go.sum create mode 100644 modules/mesh-enrolment/module.json diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go new file mode 100644 index 0000000..f23eaae --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go @@ -0,0 +1,144 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "time" +) + +// mesh-enrolment: the holder of the seat mesh-enrolment (novox/hq ADR 0587 §1), held once for the mesh. A Go bundle served +// by a runtime of its own, on its own bus account (`runs-as`). It mints and keeps no secret and holds no signing key: the +// vault mints the token and the node's account and signs (ADR 0587 §3). What it keeps is one key per open enrolment in its +// state `enrolments` (§2), and what it says is its five events. + +// Module and Seat are the module's name and the seat it holds. +const ( + Module = "mesh-enrolment" + Seat = "mesh-enrolment" + // State is the holder's bucket of open enrolments; its keys are node. and address.
(ADR 0587 §2). + State = "enrolments" +) + +// The seat's six verbs and five events (ADR 0587 §1). +var Verbs = []string{"issue", "redeem", "confirm", "revoke", "tokens", "again"} + +const ( + EventIssued = "enrolment-issued" + EventEnrolled = "node-enrolled" + EventRefused = "enrolment-refused" + EventRevoked = "enrolment-revoked" + EventExpired = "enrolment-expired" +) + +// EndEvents are the events that end an enrolment; the holder's durable consumer hears each (ADR 0587 §5) and asks the vault +// to drop the token. A seat's event is heard as .. +var EndEvents = []string{EventEnrolled, EventRefused, EventRevoked, EventExpired} + +// Stage is how far an open enrolment has come (ADR 0587 §2). +type Stage string + +const ( + // StageAsked: the operator's Approve is asked, no token exists. + StageAsked Stage = "asked" + // StageLive: the Approve is given, the token exists and expires on the bus with the key. + StageLive Stage = "live" + // StageRedeeming: the token is spent; the key has no expiry and ends only by confirm or revoke. + StageRedeeming Stage = "redeeming" +) + +const ( + // DefaultExpiry and MaxExpiry bound a token's life (ADR 0587 §4). + DefaultExpiry = time.Hour + MaxExpiry = 24 * time.Hour + // EnrolUser is the bus user a joining machine dials as: enrol. (ADR 0587 §4). + enrolPrefix = "enrol." + nodePrefix = "node." +) + +// Enrolment is the value of the key node. (ADR 0587 §2): no secret, nothing derived from the token. +type Enrolment struct { + V int `json:"v"` + Node string `json:"node"` + Address string `json:"address"` + Why string `json:"why"` + Caller string `json:"caller"` + Ask string `json:"ask,omitempty"` + Stage Stage `json:"stage"` + Issued time.Time `json:"issued"` + Expires time.Time `json:"expires"` + // KeysDigest is the digest of the public keys the first redeem presented (ADR 0587 §5), empty before it. + KeysDigest string `json:"keys-digest,omitempty"` +} + +// Reservation is the value of the key address.
: the node's name alone (ADR 0587 §2). +type Reservation struct { + V int `json:"v"` + Node string `json:"node"` +} + +// NodeKey and AddressKey are the two keys of an enrolment. +func NodeKey(node string) string { return "node." + node } +func AddressKey(address string) string { return "address." + address } + +// EnrolUser and NodeUser are the two bus users of a node: the one it joins with, and its own. +func EnrolUser(node string) string { return enrolPrefix + node } +func NodeUser(node string) string { return nodePrefix + node } + +var nodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// CheckNode says node is a node name a bus user and a key can carry. +func CheckNode(node string) error { + if !nodeName.MatchString(node) { + return fmt.Errorf("%q is not a valid node name: lowercase letters, digits and hyphens, not starting with a hyphen", node) + } + return nil +} + +func decodeEnrolment(raw []byte) (Enrolment, error) { + var e Enrolment + if err := json.Unmarshal(raw, &e); err != nil { + return Enrolment{}, fmt.Errorf("an enrolment key does not read: %w", err) + } + if e.Node == "" || e.Address == "" { + return Enrolment{}, errors.New("an enrolment key names no node or no address") + } + switch e.Stage { + case StageAsked, StageLive, StageRedeeming: + default: + return Enrolment{}, fmt.Errorf("the enrolment of %s is in no stage %q", e.Node, e.Stage) + } + return e, nil +} + +// Row is a key as the store holds it. +type Row struct { + Value []byte + Revision uint64 +} + +// ErrExists is a create refused because the key has a value; ErrChanged a write refused because the key moved. +var ( + ErrExists = errors.New("the key has a value already") + ErrChanged = errors.New("the key was written by another writer first") +) + +// Store is the holder's own state: stdio.State in the process, a map in a test. A ttl of zero is no expiry of the key's own. +type Store interface { + Get(key string) (Row, bool, error) + // Create writes a key only when it has none (ErrExists otherwise). + Create(key string, v any, ttl time.Duration) error + // Update writes a key only when it is still at revision (ErrChanged otherwise), with no expiry of its own. + Update(key string, v any, revision uint64) (uint64, error) + // Put writes a key whole, with the expiry ttl when it is not zero. + Put(key string, v any, ttl time.Duration) (uint64, error) + Delete(key string) error + Keys() ([]string, error) +} + +// Reader reads another module's state, `.`. +type Reader interface { + Keys(state string) ([]string, error) + Get(state, key string) ([]byte, bool, error) +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go new file mode 100644 index 0000000..b2490d0 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -0,0 +1,263 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Vault is the vault's side of enrolment (ADR 0587 §3, §5): the token and the node's account, and the signatures. The +// holder keeps nothing of what it answers. +type Vault interface { + // Token asks the vault to make the token and its hash for node, expiring at expires, and answers the public fields + // the joining machine must trust. No token is in the answer. + Token(node string, expires time.Time) (json.RawMessage, error) + // Sealed asks the vault for node's token sealed to the operator key, as the vault sealed it. + Sealed(node string) (json.RawMessage, error) + // Drop asks the vault to drop node's token and its hash; it answers only once both are gone. + Drop(node string) error + // Account asks for node's account sealed to the sealing key; the same node and keys again answer the same account. + Account(node, sealingKey, keysDigest string) (json.RawMessage, error) + // Retire asks the vault to retire the account it minted for node. + Retire(node string) error + // SignCertificate and SignReply sign through the vault's `sign`. + SignCertificate(node, servingKey string) (json.RawMessage, error) + SignReply(node string, reply []byte) (json.RawMessage, error) +} + +// Asker is the operator's channel as the holder asks it (asker.Client is one). +type Asker interface { + Ask(asks.Ask) error + Cancel(id string) error +} + +// Holder is the seat's holder: the state it keeps, and every other part it reaches through a port. +type Holder struct { + Store Store + Read Reader + Vault Vault + Asker Asker + // Say publishes a seat event under a message id the bus keeps one of. + Say func(event string, body any, id string) error + // AddNode asks mesh-placement to add the node (ADR 0587 §7). + AddNode func(AddNode) error + // Warrant and Act record in the hand-act log, through the seat mesh-hand-acts: the operator's warrant for an ask, and an + // act done by hand (a revoke). + Warrant func(ask string) error + Act func(what, why, cause string) error + // Loaded says whether the bus node loaded the bus user (ADR 0587 §3: redeem answers once node. is in its list). + Loaded func(user string) (bool, error) + Now func() time.Time + Logf func(string, ...any) + // AskLasts is how long the operator has to answer an issue's ask. + AskLasts time.Duration + + mu sync.Mutex + // seen is each node key's last content the watch handed over, so an expiry, which hands over no value, is said with it. + seen map[string]Enrolment + // ended is the end events this process heard, newest last: the history `tokens` answers (limit: EVENTS is not readable + // from a bundle, so a restart forgets them). + ended []json.RawMessage +} + +func (h *Holder) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *Holder) logf(f string, a ...any) { + if h.Logf != nil { + h.Logf(f, a...) + } +} + +func (h *Holder) say(event string, body any, id string) error { + if h.Say == nil { + return errors.New("the seat's events cannot be said: no publisher") + } + return h.Say(event, body, id) +} + +// AddNode is what the holder asks mesh-placement to add (ADR 0587 §7): the node, its address, the public halves of its +// keys, and the issue's warrant, which names this node and this address. +type AddNode struct { + Node string `json:"node"` + Address string `json:"address"` + Why string `json:"why"` + Keys NodeKeys `json:"keys"` + Warrant Warrant `json:"warrant"` +} + +// NodeKeys are the public halves of a joining node's keys: identity, private network, sealing and serving. +type NodeKeys struct { + Public []byte `json:"public_key"` + Overlay string `json:"overlay_key,omitempty"` + Sealing string `json:"sealing_key,omitempty"` + Serving string `json:"serving_key,omitempty"` +} + +// Warrant is the issue's warrant as the proposal carries it: the ask the operator answered, the node and the address it +// named. +type Warrant struct { + Ask string `json:"ask"` + Node string `json:"node"` + Address string `json:"address"` +} + +// Digest is the digest of the keys a node presented, the one hash an event carries (ADR 0587 §1). +func (k NodeKeys) Digest() string { return delivery.DigestOf(k.bytes()) } + +func (k NodeKeys) bytes() []byte { + return []byte("novox-mesh-enrol-keys\x00" + string(k.Public) + "\x00" + k.Overlay + "\x00" + k.Sealing + "\x00" + k.Serving) +} + +// issuing says whether issue is on: only while the controller's lease is handed to a holder that is no controller (ADR 0587 +// §4), and why either way. +func (h *Holder) issuing() (bool, string) { + raw, found, err := h.Read.Get("mesh-controller.lease", buscompose.ControllerLeaseKey) + return buscompose.LeaseHandedAway(raw, found, err) +} + +// openEnrolments reads every node key of the bucket. +func (h *Holder) openEnrolments() (map[string]Row, map[string]Enrolment, error) { + keys, err := h.Store.Keys() + if err != nil { + return nil, nil, fmt.Errorf("the enrolments cannot be listed: %w", err) + } + sort.Strings(keys) + rows, out := map[string]Row{}, map[string]Enrolment{} + for _, k := range keys { + node, ok := strings.CutPrefix(k, "node.") + if !ok { + continue + } + row, found, err := h.Store.Get(k) + if err != nil { + return nil, nil, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err) + } + if !found { + continue + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", k, err) + } + rows[node], out[node] = row, e + } + return rows, out, nil +} + +// placements reads every node's placement, assembled from its keys. +func (h *Holder) placements() (map[string]delivery.Placement, error) { + keys, err := h.Read.Keys("mesh-delivery-assigner.placement") + if err != nil { + return nil, fmt.Errorf("placement cannot be listed: %w", err) + } + held := map[string][]byte{} + for _, k := range keys { + if raw, found, err := h.Read.Get("mesh-delivery-assigner.placement", k); err != nil { + return nil, fmt.Errorf("placement key %s cannot be read: %w", k, err) + } else if found { + held[k] = raw + } + } + p, err := delivery.PlacementsOf(held) + if len(p) == 0 && err != nil { + return nil, err + } + return p, nil +} + +// hasRecord says the node already writes a record (a node of the mesh, ADR 0587 §4: "a node record ... holds it"). +func (h *Holder) hasRecord(node string) bool { + keys, err := h.Read.Keys("node-engine.have") + if err != nil { + return false + } + for _, k := range keys { + if n, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode && n == node { + return true + } + } + return false +} + +// nextAddress is the lowest address of the private network that no placement key and no reservation holds (ADR 0587 §4), +// after skip. The network is the hub's range; the hub's own address and the tunnel's are never given. +func nextAddress(placements map[string]delivery.Placement, reserved map[string]bool, skip map[string]bool) (string, error) { + var prefix netip.Prefix + taken := map[string]bool{} + for _, p := range placements { + if a := p.Decided.Address; a != "" { + taken[a] = true + } + if t := p.Decided.Tunnel; t != nil { + if t.Address != "" { + if a, err := netip.ParsePrefix(t.Address); err == nil { + taken[a.Addr().String()] = true + } else { + taken[t.Address] = true + } + } + if t.Range != "" && p.Decided.Hub && !prefix.IsValid() { + if r, err := netip.ParsePrefix(t.Range); err == nil { + prefix = r.Masked() + } + } + } + } + if !prefix.IsValid() { + for _, p := range placements { + if a, err := netip.ParseAddr(p.Decided.Address); err == nil && p.Decided.Hub && a.Is4() { + prefix = netip.PrefixFrom(a, 24).Masked() + } + } + } + if !prefix.IsValid() { + return "", errors.New("no private network is known: the hub's placement names no range or address") + } + for a := prefix.Addr().Next(); prefix.Contains(a); a = a.Next() { + s := a.String() + if prefix.Addr().Is4() && !prefix.Contains(a.Next()) { + break // the broadcast address + } + if taken[s] || reserved[s] || skip[s] { + continue + } + return s, nil + } + return "", fmt.Errorf("the private network %s has no free address", prefix) +} + +func (h *Holder) reservedAddresses() (map[string]bool, error) { + keys, err := h.Store.Keys() + if err != nil { + return nil, fmt.Errorf("the enrolments cannot be listed: %w", err) + } + out := map[string]bool{} + for _, k := range keys { + if a, ok := strings.CutPrefix(k, "address."); ok { + out[a] = true + } + } + return out, nil +} + +// lifetimeOf is the time a key lives on the bus for an enrolment that expires at expires. +func (h *Holder) lifetimeOf(expires time.Time) time.Duration { + if d := expires.Sub(h.now()); d > time.Second { + return d + } + return time.Second +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go new file mode 100644 index 0000000..e907b6b --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -0,0 +1,240 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// memStore is the holder's bucket in memory: revisions, and the expiry each key was written with. +type memStore struct { + mu sync.Mutex + rows map[string]Row + ttl map[string]time.Duration + rev uint64 + // failDelete names keys whose delete fails once. + failDelete map[string]bool +} + +func newMem() *memStore { + return &memStore{rows: map[string]Row{}, ttl: map[string]time.Duration{}, failDelete: map[string]bool{}} +} + +func (m *memStore) write(key string, v any, ttl time.Duration) uint64 { + raw, _ := json.Marshal(v) + m.rev++ + m.rows[key] = Row{Value: raw, Revision: m.rev} + m.ttl[key] = ttl + return m.rev +} + +func (m *memStore) Get(key string) (Row, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + r, ok := m.rows[key] + return r, ok, nil +} + +func (m *memStore) Create(key string, v any, ttl time.Duration) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.rows[key]; ok { + return ErrExists + } + m.write(key, v, ttl) + return nil +} + +func (m *memStore) Update(key string, v any, revision uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if r, ok := m.rows[key]; !ok || r.Revision != revision { + return 0, ErrChanged + } + return m.write(key, v, 0), nil +} + +func (m *memStore) Put(key string, v any, ttl time.Duration) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.write(key, v, ttl), nil +} + +func (m *memStore) Delete(key string) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.failDelete[key] { + delete(m.failDelete, key) + return errors.New("the bus did not answer") + } + delete(m.rows, key) + delete(m.ttl, key) + return nil +} + +func (m *memStore) Keys() ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for k := range m.rows { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// memRead is other modules' states, `.` then key. +type memRead map[string]map[string]string + +func (r memRead) Keys(state string) ([]string, error) { + var out []string + for k := range r[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (r memRead) Get(state, key string) ([]byte, bool, error) { + v, ok := r[state][key] + return []byte(v), ok, nil +} + +// fakeVault records its calls in order and fails on the verbs named in fail, once each. +type fakeVault struct { + calls []string + fail map[string]bool +} + +func (v *fakeVault) do(what string) error { + v.calls = append(v.calls, what) + if v.fail[what] { + delete(v.fail, what) + return errors.New("the vault did not answer " + what) + } + return nil +} + +func (v *fakeVault) Token(node string, expires time.Time) (json.RawMessage, error) { + return json.RawMessage(`{"broker":"b","fingerprint":"f","signer":"s"}`), v.do("token " + node) +} +func (v *fakeVault) Sealed(node string) (json.RawMessage, error) { + return json.RawMessage(`"sealed-for-` + node + `"`), v.do("sealed " + node) +} +func (v *fakeVault) Drop(node string) error { return v.do("drop " + node) } +func (v *fakeVault) Account(node, sealing, digest string) (json.RawMessage, error) { + return json.RawMessage(`{"account":"` + node + `"}`), v.do("account " + node) +} +func (v *fakeVault) Retire(node string) error { return v.do("retire " + node) } +func (v *fakeVault) SignCertificate(node, serving string) (json.RawMessage, error) { + return json.RawMessage(`{"certificate":"c"}`), v.do("certificate " + node) +} +func (v *fakeVault) SignReply(node string, reply []byte) (json.RawMessage, error) { + return json.RawMessage(`{"signature":"sig","signer":"fp"}`), v.do("sign " + node) +} + +type fakeAsker struct { + asked, cancelled []string + err error +} + +func (a *fakeAsker) Ask(k asks.Ask) error { + if a.err != nil { + return a.err + } + a.asked = append(a.asked, k.ID) + return nil +} +func (a *fakeAsker) Cancel(id string) error { a.cancelled = append(a.cancelled, id); return nil } + +type said struct { + Event, ID string + Body json.RawMessage +} + +// rig is a holder over fakes, with the lease handed to mesh-grants, one hub in placement and a clock the test moves. +type rig struct { + t *testing.T + h *Holder + st *memStore + vault *fakeVault + asker *fakeAsker + rd memRead + now time.Time + said []said + added []AddNode + acts []string + warr []string + loaded bool + placeErr error +} + +const hubPlacement = `{"v":1,"node":"hub","decided":{"hub":true,"address":"10.9.0.1","tunnel":{"interface":"wg0","address":"10.9.0.1/24","range":"10.9.0.0/24"}},"modules":[]}` + +func newRig(t *testing.T) *rig { + r := &rig{t: t, st: newMem(), vault: &fakeVault{fail: map[string]bool{}}, asker: &fakeAsker{}, now: time.Date(2026, 11, 2, 10, 0, 0, 0, time.UTC), loaded: true} + r.rd = memRead{ + "mesh-controller.lease": {"holder": `{"handed-to":"mesh-grants","handed-by":"operator"}`}, + "mesh-delivery-assigner.placement": {"hub.node": hubPlacement}, + "node-engine.have": {}, + } + r.h = &Holder{Store: r.st, Read: r.rd, Vault: r.vault, Asker: r.asker, AskLasts: time.Hour, + Say: func(event string, body any, id string) error { + raw, _ := json.Marshal(body) + r.said = append(r.said, said{event, id, raw}) + return nil + }, + AddNode: func(a AddNode) error { r.added = append(r.added, a); return r.placeErr }, + Warrant: func(ask string) error { r.warr = append(r.warr, ask); return nil }, + Act: func(what, why, cause string) error { r.acts = append(r.acts, what); return nil }, + Loaded: func(string) (bool, error) { return r.loaded, nil }, + Now: func() time.Time { return r.now }, + Logf: func(f string, a ...any) { t.Logf(f, a...) }} + return r +} + +func (r *rig) events() []string { + var out []string + for _, s := range r.said { + out = append(out, s.Event) + } + return out +} + +func (r *rig) enrolment(node string) (Enrolment, bool) { + row, ok, _ := r.st.Get(NodeKey(node)) + if !ok { + return Enrolment{}, false + } + e, err := decodeEnrolment(row.Value) + if err != nil { + r.t.Fatal(err) + } + return e, true +} + +func (r *rig) keys() []string { k, _ := r.st.Keys(); return k } + +func (r *rig) has(key string) bool { return slices.Contains(r.keys(), key) } + +func mustContain(t *testing.T, got, want string) { + t.Helper() + if !strings.Contains(got, want) { + t.Fatalf("%q does not contain %q", got, want) + } +} + +func errText(err error) string { + if err == nil { + return "" + } + return fmt.Sprint(err) +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go new file mode 100644 index 0000000..189f172 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -0,0 +1,146 @@ +// mesh-enrolment: the holder of the seat mesh-enrolment (novox/hq ADR 0587). A Go bundle served by a runtime of its own, +// on its own bus account (`runs-as`), so that its account alone writes its bucket `enrolments` with per-key expiry. It +// asks the operator on their channel, the vault to mint and sign, mesh-placement to add the node and mesh-hand-acts to +// record the acts. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the holder's bucket, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(key string) (Row, bool, error) { + e, err := stdio.State(State).Get(key) + if err != nil || e == nil { + return Row{}, false, err + } + return Row{Value: e.Value, Revision: e.Revision}, true, nil +} + +func (runtimeStore) Create(key string, v any, ttl time.Duration) error { + var err error + if ttl > 0 { + _, err = stdio.State(State).CreateTTL(key, v, ttl) + } else { + _, err = stdio.State(State).Create(key, v) + } + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Update(key string, v any, revision uint64) (uint64, error) { + rev, err := stdio.State(State).Update(key, v, revision) + if errors.Is(err, stdio.ErrChanged) { + return 0, ErrChanged + } + return rev, err +} + +func (runtimeStore) Put(key string, v any, ttl time.Duration) (uint64, error) { + if ttl > 0 { + return stdio.State(State).PutTTL(key, v, ttl) + } + return stdio.State(State).Put(key, v) +} + +func (runtimeStore) Delete(key string) error { return stdio.State(State).Delete(key) } +func (runtimeStore) Keys() ([]string, error) { return stdio.State(State).Keys() } + +// runtimeReader reads another module's state through the runtime. +type runtimeReader struct{} + +func (runtimeReader) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeReader) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// SeatEventSubject is where the seat says an event: mesh.seat..event.. +func SeatEventSubject(event string) string { return "mesh.seat." + Seat + ".event." + event } + +// newHolder is the holder over the runtime. +func newHolder() *Holder { + call := Call(stdio.Ask) + rec := handacts.Recorder{Ask: stdio.Ask} + rd := runtimeReader{} + return &Holder{ + Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call}, Asker: asker.New("asked"), + Say: func(event string, body any, id string) error { + return stdio.SeatPublish(SeatEventSubject(event), body, id) + }, + AddNode: addNodeOver(call), + Warrant: func(ask string) error { _, err := rec.RecordWarrant(Module, ask); return err }, + Act: func(what, why, cause string) error { + _, err := rec.Record(handacts.Request{What: what, Why: why, Cause: cause}) + return err + }, + Loaded: loadedVia(rd), + Now: time.Now, + Logf: func(f string, a ...any) { logf("["+Module+"] "+f, a...) }, + AskLasts: 55 * time.Minute, + } +} + +// loadedVia says a bus user is loaded when the grants part's last pass lists it among the composed users and the bus node's +// loaded list holds it (its credentials row says loaded). +func loadedVia(rd Reader) func(string) (bool, error) { + return func(user string) (bool, error) { + raw, found, err := rd.Get("mesh-grants.user-list", "pass") + if err != nil { + return false, err + } + if !found { + return false, errors.New("the grants part has composed no list yet") + } + var st struct { + Compare struct { + Unread string `json:"unread"` + } `json:"compare"` + Credentials struct { + Users []struct { + User string `json:"user"` + InShadow bool `json:"in_shadow"` + Loaded bool `json:"loaded"` + } `json:"users"` + } `json:"credentials"` + } + if err := json.Unmarshal(raw, &st); err != nil { + return false, fmt.Errorf("the grants part's list state does not read: %w", err) + } + if st.Compare.Unread != "" { + return false, errors.New("the bus node's loaded list cannot be read: " + st.Compare.Unread) + } + for _, u := range st.Credentials.Users { + if u.User == user { + return u.InShadow && u.Loaded, nil + } + } + return false, nil + } +} + +func main() { + h := newHolder() + if err := stdio.Serve("", Tools(h)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go new file mode 100644 index 0000000..df23f1e --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go @@ -0,0 +1,118 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "time" +) + +// TokenRow is one open enrolment as `tokens` answers it (ADR 0587 §8): its stage, expiry, ask and revision, and for a node +// at stage live the token sealed to the operator key as the vault sealed it (the holder never reads it and keeps it nowhere). +type TokenRow struct { + Node string `json:"node"` + Address string `json:"address"` + Why string `json:"why"` + Caller string `json:"caller"` + Stage Stage `json:"stage"` + Ask string `json:"ask,omitempty"` + Issued time.Time `json:"issued"` + Expires time.Time `json:"expires"` + Age string `json:"age"` + Revision uint64 `json:"revision"` + Keys string `json:"keys-digest,omitempty"` + Sealed json.RawMessage `json:"sealed,omitempty"` + // Unsealed says why no sealed token is given for a node that should have one. + Unsealed string `json:"unsealed,omitempty"` +} + +// Issuing is whether issue is on, and why. +type Issuing struct { + On bool `json:"on"` + Why string `json:"why"` +} + +// Fill is how full the bucket is: its keys and the bytes of their values (the state's cap is the bus's, which no bundle +// can read, so the fill is stated without it). +type Fill struct { + Keys int `json:"keys"` + Bytes int `json:"bytes"` +} + +// TokensAnswer is what `tokens` answers. +type TokensAnswer struct { + Open []TokenRow `json:"open"` + Issuing Issuing `json:"issuing"` + Fill Fill `json:"fill"` + History []json.RawMessage `json:"history,omitempty"` + // HistoryFrom says where the history comes from, when one was asked for. + HistoryFrom string `json:"history-from,omitempty"` +} + +// Tokens answers one open enrolment or all, with the bucket's fill, whether issuing is on and why, and with history the +// ended enrolments the holder's own events say (ADR 0587 §8). +func (h *Holder) Tokens(node string, history bool) (TokensAnswer, error) { + rows, open, err := h.openEnrolments() + if err != nil { + return TokensAnswer{}, err + } + var out TokensAnswer + on, why := h.issuing() + out.Issuing = Issuing{On: on, Why: why} + if keys, err := h.Store.Keys(); err == nil { + out.Fill.Keys = len(keys) + for _, k := range keys { + if r, found, err := h.Store.Get(k); err == nil && found { + out.Fill.Bytes += len(r.Value) + } + } + } + if node != "" { + if _, ok := open[node]; !ok && !history { + return TokensAnswer{}, fmt.Errorf("no enrolment of %s is open: tokens without a node lists those that are", node) + } + } + names := make([]string, 0, len(open)) + for n := range open { + if node == "" || n == node { + names = append(names, n) + } + } + sort.Strings(names) + out.Open = []TokenRow{} + for _, n := range names { + e := open[n] + row := TokenRow{Node: e.Node, Address: e.Address, Why: e.Why, Caller: e.Caller, Stage: e.Stage, Ask: e.Ask, + Issued: e.Issued, Expires: e.Expires, Age: h.now().Sub(e.Issued).Round(time.Second).String(), + Revision: rows[n].Revision, Keys: e.KeysDigest} + if e.Stage == StageLive && node != "" { + // Asked of the vault at this moment and relayed as the vault sealed it: kept nowhere here (ADR 0587 §8). + sealed, err := h.Vault.Sealed(n) + if err != nil { + row.Unsealed = "the vault did not give the token: " + err.Error() + } else { + row.Sealed = sealed + } + } + out.Open = append(out.Open, row) + } + if history { + h.mu.Lock() + for _, e := range h.ended { + if node == "" || endedNode(e) == node { + out.History = append(out.History, e) + } + } + h.mu.Unlock() + out.HistoryFrom = "the end events this holder heard since it started; the bus keeps them 7 days, and a bundle cannot read that stream" + } + return out, nil +} + +func endedNode(raw json.RawMessage) string { + var e struct { + Node string `json:"node"` + } + _ = json.Unmarshal(raw, &e) + return e.Node +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go new file mode 100644 index 0000000..a200701 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + "time" +) + +// seed writes an open enrolment of node at stage, with its address key, as the holder would have. +func (r *rig) seed(node, address string, stage Stage) { + e := Enrolment{V: 1, Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, Stage: stage, + Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour)} + if stage == StageRedeeming { + e.KeysDigest = "sha256:keys" + } + _, _ = r.st.Put(NodeKey(node), e, 0) + _, _ = r.st.Put(AddressKey(address), Reservation{V: 1, Node: node}, 0) +} + +// 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, the fill and whether issuing is on; +// for one node at live it relays the vault's sealed token and keeps nothing; and the manifest says what the code does. +func TestTokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageAsked) + r.seed("g14", "10.9.0.3", StageLive) + r.seed("novox", "10.9.0.4", StageRedeeming) + all, err := r.h.Tokens("", false) + if err != nil { + t.Fatal(err) + } + var stages []string + for _, row := range all.Open { + stages = append(stages, row.Node+":"+string(row.Stage)) + if row.Revision == 0 || row.Expires.IsZero() || row.Ask == "" || row.Sealed != nil { + t.Errorf("row %+v lacks revision, expiry or ask, or relays a token for a list of all", row) + } + } + if !slices.Equal(stages, []string{"ace:asked", "g14:live", "novox:redeeming"}) || all.Fill.Keys != 6 || !all.Issuing.On { + t.Errorf("tokens %v, fill %+v, issuing %+v", stages, all.Fill, all.Issuing) + } + one, err := r.h.Tokens("g14", false) + if err != nil || len(one.Open) != 1 || string(one.Open[0].Sealed) != `"sealed-for-g14"` { + t.Fatalf("one node: %+v, %v", one, err) + } + for k, row := range r.st.rows { + if slices.Contains(r.vault.calls, "sealed g14") && string(row.Value) != "" { + var any map[string]json.RawMessage + _ = json.Unmarshal(row.Value, &any) + if _, has := any["sealed"]; has { + t.Errorf("the key %s holds a sealed token", k) + } + } + } + // Before the lease write issuing is off, and says why. + r.rd["mesh-controller.lease"]["holder"] = `{"instance":"novox.mesh-controller"}` + off, _ := r.h.Tokens("", false) + mustContain(t, off.Issuing.Why, "controller holds the lease") + if off.Issuing.On { + t.Error("issuing is on while the controller holds the lease") + } +} + +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Runs string `json:"runs-as"` + Consumes []string `json:"consumes"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.Runs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || + !slices.Equal(m.Claims[0].Serves, Verbs) || len(m.Emits) != 0 { + t.Errorf("module %q runs as %q, claims %+v, emits %v", m.Module, m.Runs, m.Claims, m.Emits) + } + var heard []string + for _, e := range EndEvents { + heard = append(heard, Seat+"."+e) + } + if !slices.Equal(m.Consumes, heard) { + t.Errorf("consumes %v, the end events are %v", m.Consumes, heard) + } + if len(m.State) != 2 || string(m.State[0]) == "" || !slices.ContainsFunc(m.State, func(s json.RawMessage) bool { + var d struct { + Name string `json:"name"` + TTL bool `json:"key-ttl"` + } + return json.Unmarshal(s, &d) == nil && d.Name == State && d.TTL + }) { + t.Errorf("state %s: enrolments must allow a time to live per key", m.State) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go new file mode 100644 index 0000000..652bbc4 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -0,0 +1,51 @@ +package main + +import ( + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Tool names: the seat's verbs, as the manifest claims them (ADR 0587 §1, §8). +const ( + TokensTool = Seat + ".tokens" +) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +func arg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func flag(a map[string]any, k string) bool { + return a[k] == true || strings.EqualFold(arg(a, k), "true") +} + +var _ = time.Second + +// Tools are what the runtime serves. +func Tools(h *Holder) []stdio.Tool { + return []stdio.Tool{ + {Name: TokensTool, + Description: "What the seat holds (novox/hq ADR 0587 §8): every open enrolment, or one node's, with its stage (asked, live or " + + "redeeming), address, expiry, ask, revision and age; whether issuing is on and why (it is on only from the controller's " + + "lease write); the bucket's fill. With a node at stage live, the token sealed to the operator key, asked of the vault at " + + "this moment and relayed as the vault sealed it: the one route by which the operator gets a token, which this seat never " + + "reads and keeps nowhere. With history=true the ended enrolments its events say. A machine stuck at redeeming shows here " + + "with its age; revoke ends it. Read-only.", + Input: obj(map[string]any{"node": str("one node"), "history": str("\"true\": the ended enrolments")}), + Run: func(a map[string]any) (any, error) { + return h.Tokens(arg(a, "node"), flag(a, "history")) + }}, + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go new file mode 100644 index 0000000..b96269f --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Seats the holder calls (ADR 0587 §3, §5, §7). The vault's two verbs are the ones its manifest names this seat the only +// caller of; its `sign` names this seat for a node's certificate and a redeem answer. +const ( + vaultToken = "seat:mesh-vault.enrolment-token" + vaultAccount = "seat:mesh-vault.node-account" + vaultSign = "seat:mesh-vault.sign" + placementAdd = "seat:mesh-placement.propose" +) + +// Call asks a seat verb by its address (stdio.Ask through the runtime, or a test's). +type Call func(key string, body any) (json.RawMessage, error) + +// vaultOver is the vault as the holder reaches it: the seat's verbs over a Call. +type vaultOver struct{ call Call } + +func (v vaultOver) Token(node string, expires time.Time) (json.RawMessage, error) { + return v.call(vaultToken, map[string]any{"node": node, "expires": expires.UTC().Format(time.RFC3339)}) +} + +func (v vaultOver) Sealed(node string) (json.RawMessage, error) { + return v.call(vaultToken, map[string]any{"node": node, "sealed": true}) +} + +func (v vaultOver) Drop(node string) error { + _, err := v.call(vaultToken, map[string]any{"node": node, "drop": true}) + return err +} + +func (v vaultOver) Account(node, sealingKey, keysDigest string) (json.RawMessage, error) { + return v.call(vaultAccount, map[string]any{"node": node, "sealing_key": sealingKey, "keys_digest": keysDigest}) +} + +func (v vaultOver) Retire(node string) error { + _, err := v.call(vaultAccount, map[string]any{"node": node, "retire": true}) + return err +} + +func (v vaultOver) SignCertificate(node, servingKey string) (json.RawMessage, error) { + return v.call(vaultSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node, "serving_key": servingKey}) +} + +func (v vaultOver) SignReply(node string, reply []byte) (json.RawMessage, error) { + return v.call(vaultSign, map[string]any{"kind": delivery.KindEnrolment, "node": node, + "reply": base64.StdEncoding.EncodeToString(reply)}) +} + +// addNodeOver asks mesh-placement to add the node with the issue's warrant (ADR 0587 §7). +func addNodeOver(call Call) func(AddNode) error { + return func(a AddNode) error { + _, err := call(placementAdd, map[string]any{ + "why": a.Why, + "changes": []any{map[string]any{"op": "add-node", "node": a.Node, "address": a.Address, "keys": a.Keys, "warrant": a.Warrant}}, + }) + if err != nil { + return fmt.Errorf("mesh-placement did not take the node %s: %w", a.Node, err) + } + return nil + } +} diff --git a/modules/mesh-enrolment/go.mod b/modules/mesh-enrolment/go.mod new file mode 100644 index 0000000..03fe8f2 --- /dev/null +++ b/modules/mesh-enrolment/go.mod @@ -0,0 +1,16 @@ +module mesh-enrolment + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/mesh-enrolment/go.sum b/modules/mesh-enrolment/go.sum new file mode 100644 index 0000000..1c91827 --- /dev/null +++ b/modules/mesh-enrolment/go.sum @@ -0,0 +1,26 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-enrolment/module.json b/modules/mesh-enrolment/module.json new file mode 100644 index 0000000..4504a55 --- /dev/null +++ b/modules/mesh-enrolment/module.json @@ -0,0 +1,88 @@ +{ + "module": "mesh-enrolment", + "version": "1", + "runs-as": "mesh-enrolment", + "claims": [ + { + "name": "mesh-enrolment", + "scope": "mesh", + "serves": [ + "issue", + "redeem", + "confirm", + "revoke", + "tokens", + "again" + ] + } + ], + "consumes": [ + "mesh-enrolment.node-enrolled", + "mesh-enrolment.enrolment-refused", + "mesh-enrolment.enrolment-revoked", + "mesh-enrolment.enrolment-expired" + ], + "uses": [ + "operator-channel" + ], + "invokes": [ + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", + "seat:mesh-placement.propose", + "seat:mesh-vault.enrolment-token", + "seat:mesh-vault.node-account", + "seat:mesh-vault.sign" + ], + "reads": [ + "mesh-controller.lease", + "mesh-delivery-assigner.placement", + "mesh-grants.user-list", + "node-engine.have" + ], + "state": [ + { + "name": "enrolments", + "key-ttl": true + }, + { + "name": "asked", + "history": 5 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-enrolment", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-enrolment", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-enrolment" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-enrolment", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-enrolment", + "binary": "mesh-enrolment", + "loads": [ + "mesh-enrolment" + ] + } + ] + } +} From 4edd0e526cdaa013c3211cad91722e131a1d3de2 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:19:15 +0100 Subject: [PATCH 02/20] mesh-enrolment: issue, the operator's ask and the warrant, and again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit issue refuses at once naming every finding (a bad name, a name a node record, a placement key or an open enrolment holds, an expiry past a day, any call before the lease write), reserves the lowest free address by creating its key, writes the enrolment at asked and asks the operator (ADR 0587 §4). On the yes the warrant is recorded, the vault makes the token and both keys go live with the bus's expiry; a no or an expired ask takes both keys and says enrolment-refused. Tests added: three table-style tests; none deleted here (paid with 774/S59's merges). (issue 774, 774/S55) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/issue.go | 306 ++++++++++++++++++ .../cmd/mesh-enrolment/issue_test.go | 133 ++++++++ .../cmd/mesh-enrolment/kit_test.go | 13 +- .../cmd/mesh-enrolment/tools.go | 37 ++- 4 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/issue.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go new file mode 100644 index 0000000..e6f6be2 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -0,0 +1,306 @@ +package main + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// Event is what each of the seat's events carries (ADR 0587 §2): the key's whole content, which holds no secret and nothing +// derived from the token, and for an end the reason. Node is repeated at the top so a consumer reads it without the rest. +type Event struct { + Node string `json:"node"` + Enrolment Enrolment `json:"enrolment"` + Reason string `json:"reason,omitempty"` + // Issued names the enrolment-issued event that carried the content (for enrolment-expired, whose key is gone). + Issued string `json:"issued-event,omitempty"` +} + +// eventID is the message id an event of e is said under: the bus keeps one message of it, so an event said again after a +// crash is not said twice. +func eventID(event string, e Enrolment) string { + return event + "." + e.Node + "." + strconv.FormatInt(e.Issued.UnixNano(), 10) +} + +func (h *Holder) sayEvent(event string, e Enrolment, reason string) error { + return h.say(event, Event{Node: e.Node, Enrolment: e, Reason: reason}, eventID(event, e)) +} + +// IssueAnswer is what `issue` answers: the ask is the operator's to answer, and nothing else happens until they do. +type IssueAnswer struct { + Node string `json:"node"` + Address string `json:"address"` + Ask string `json:"ask"` + Expires time.Time `json:"expires"` + Says string `json:"says"` +} + +// Issue opens an enrolment (ADR 0587 §4): it refuses at once, naming every finding, a name that is no node name or that a +// node record, a placement key or an open enrolment holds, and every call before the lease write; chooses the lowest free +// address and reserves it by creating its key (the bus refuses a key that exists, so two issues never get one address); +// writes the key at stage asked; and asks the operator, on a channel that proves who answered. The token is made only on +// the operator's yes (Warranted). +func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnswer, error) { + var findings []string + if caller == "" || caller == stdio.Unproven { + findings = append(findings, "the caller is not proven by the runtime, and an enrolment records who asked") + } + if err := CheckNode(node); err != nil { + findings = append(findings, err.Error()) + } + if strings.TrimSpace(why) == "" { + findings = append(findings, "no why is given: the operator is asked why this machine joins") + } + if lasts == 0 { + lasts = DefaultExpiry + } + if lasts < 0 || lasts > MaxExpiry { + findings = append(findings, fmt.Sprintf("the expiry %s is outside zero to %s", lasts, MaxExpiry)) + } + if on, reason := h.issuing(); !on { + findings = append(findings, "issuing is off until the controller's lease is written to a holder that is no controller: "+reason+ + "; until then the controller's `token issue` is the only issuer (ADR 0587 §4)") + } + placements, perr := h.placements() + if perr != nil { + findings = append(findings, "placement cannot be read: "+perr.Error()) + } + if CheckNode(node) == nil { + if _, ok := placements[node]; ok { + findings = append(findings, node+" has a placement key already") + } + if h.hasRecord(node) { + findings = append(findings, node+" has a node record already") + } + if _, found, err := h.Store.Get(NodeKey(node)); err != nil { + findings = append(findings, "the enrolments cannot be read: "+err.Error()) + } else if found { + findings = append(findings, "an enrolment of "+node+" is open already: revoke it first") + } + } + if len(findings) > 0 { + return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %s", node, strings.Join(findings, "; ")) + } + now := h.now() + e := Enrolment{V: 1, Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, Issued: now, Expires: now.Add(lasts)} + e.Ask = "enrol-" + node + "-" + now.UTC().Format("20060102t150405") + skip := map[string]bool{} + for { + reserved, err := h.reservedAddresses() + if err != nil { + return IssueAnswer{}, err + } + addr, err := nextAddress(placements, reserved, skip) + if err != nil { + return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %w", node, err) + } + switch err := h.Store.Create(AddressKey(addr), Reservation{V: 1, Node: node}, 0); { + case err == nil: + e.Address = addr + case errors.Is(err, ErrExists): + skip[addr] = true // another issue reserved it first: the next + continue + default: + return IssueAnswer{}, fmt.Errorf("the address %s cannot be reserved: %w", addr, err) + } + break + } + if err := h.Store.Create(NodeKey(node), e, 0); err != nil { + _ = h.Store.Delete(AddressKey(e.Address)) + if errors.Is(err, ErrExists) { + return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: an enrolment of it was opened meanwhile", node) + } + return IssueAnswer{}, fmt.Errorf("the enrolment of %s cannot be written: %w", node, err) + } + if err := h.raise(e); err != nil { + h.release(e) + return IssueAnswer{}, err + } + return IssueAnswer{Node: node, Address: e.Address, Ask: e.Ask, Expires: e.Expires, + Says: "the operator is asked on their channel; only their yes makes the token. mesh-enrolment.tokens shows where it stands"}, nil +} + +// release takes an enrolment's two keys away, the address key last (the node's key is what names the reservation's owner). +func (h *Holder) release(e Enrolment) { + if err := h.Store.Delete(NodeKey(e.Node)); err != nil { + h.logf("the key of %s could not be deleted and stays: %v", e.Node, err) + } + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + h.logf("the address key %s could not be deleted and stays: %v", e.Address, err) + } +} + +// AskOf is the ask of an enrolment, in the words the operator reads: that a machine of this name may join until the +// expiry at this address. Its yes binds the act (name, address, expiry). +func (h *Holder) AskOf(e Enrolment) (asks.Ask, error) { + act := asks.Act{"verb": Seat + ".issue", "module": Module, "node": e.Node, "address": e.Address, + "expires": e.Expires.UTC().Format(time.RFC3339)} + bind, err := asks.ActDigest(act) + if err != nil { + return asks.Ask{}, err + } + until := e.Expires.UTC().Format("2006-01-02 15:04 MST") + closes := h.now().Add(h.AskLasts) + if closes.After(e.Expires) { + closes = e.Expires + } + return asks.Ask{ID: e.Ask, Headline: "Let the machine " + e.Node + " join the mesh?", + Explanation: "A machine named " + e.Node + " may join the mesh until " + until + ", at the private address " + e.Address + ".\n" + + "Your yes lets the vault make a one-time token for it; you open the token at your own desk and type it on the new machine.\n" + + "What the machine runs is placed later, as for any machine.", + Who: asks.Operator, + Options: []asks.Option{{ID: "yes", Label: "Yes, let it join", Does: "The vault makes a one-time token for " + e.Node + ", valid until " + until + ".", Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No", Does: "Nothing is made; the address is free again.", Level: asks.Acknowledge}}, + Expires: closes, OnExpiry: "No token is made and the address is free again.", About: "mesh-enrolment-" + e.Node, + Details: "Asked by " + e.Caller + ": " + e.Why}, nil +} + +// raise asks the operator; a question that cannot be asked is taken back at the router, since a publish that timed out may +// have reached it, so a yes cannot land on an enrolment the holder has forgotten. +func (h *Holder) raise(e Enrolment) error { + ask, err := h.AskOf(e) + if err != nil { + return err + } + if err := h.Asker.Ask(ask); err != nil { + if cerr := h.Asker.Cancel(ask.ID); cerr != nil { + h.logf("the ask %s that could not be published was not taken back at the router: %v", ask.ID, cerr) + } + return fmt.Errorf("the operator could not be asked about %s: %w", e.Node, err) + } + return nil +} + +// byAsk finds the open enrolment an ask belongs to. +func (h *Holder) byAsk(ask string) (Enrolment, Row, bool, error) { + rows, open, err := h.openEnrolments() + if err != nil { + return Enrolment{}, Row{}, false, err + } + for n, e := range open { + if e.Ask == ask { + return e, rows[n], true, nil + } + } + return Enrolment{}, Row{}, false, nil +} + +// Warranted is the operator's answer, handed once by the asker (asker.Client.Listen): a no ends the enrolment; a yes records +// the warrant, asks the vault for the token, moves the key to live with the bus's expiry at the token's, and says +// enrolment-issued (ADR 0587 §4, §5). +func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { + e, _, found, err := h.byAsk(w.Ask) + if err != nil { + return err + } + if !found { + return fmt.Errorf("the warrant answers the ask %s, which no open enrolment holds: nothing is made", w.Ask) + } + if e.Stage != StageAsked { + return nil // answered already + } + if o.ID != "yes" { + return h.refuse(e, "the operator declined") + } + if !h.now().Before(e.Expires) { + return h.refuse(e, "the enrolment's expiry passed before the operator answered") + } + if h.Warrant != nil { + if err := h.Warrant(w.Ask); err != nil { + return fmt.Errorf("the warrant %s is not recorded in the hand-act log, so no token is made (mesh-enrolment.again asks again): %w", w.Ask, err) + } + } + if _, err := h.Vault.Token(e.Node, e.Expires); err != nil { + return fmt.Errorf("the vault did not make the token of %s (mesh-enrolment.again asks the operator again): %w", e.Node, err) + } + e.Stage = StageLive + ttl := h.lifetimeOf(e.Expires) + if _, err := h.Store.Put(NodeKey(e.Node), e, ttl); err != nil { + return fmt.Errorf("the enrolment of %s cannot be moved to live: %w", e.Node, err) + } + if _, err := h.Store.Put(AddressKey(e.Address), Reservation{V: 1, Node: e.Node}, ttl); err != nil { + return fmt.Errorf("the address key %s cannot be given its expiry: %w", e.Address, err) + } + return h.sayEvent(EventIssued, e, "") +} + +// refuse ends an enrolment at stage asked: both keys go and enrolment-refused carries the key's content (ADR 0587 §4). +func (h *Holder) refuse(e Enrolment, reason string) error { + if err := h.sayEvent(EventRefused, e, reason); err != nil { + return fmt.Errorf("enrolment-refused of %s cannot be said, so its keys stay: %w", e.Node, err) + } + h.release(e) + return nil +} + +// Said is the asker's note of an ask that ended with no yes or no (expired, or refused by the router): the enrolment ends. +func (h *Holder) Said(hd asker.Handled, w asks.Warrant, err error) { + if err != nil { + h.logf("the answer to %s (%s) ended in an error: %v", w.Ask, hd, err) + } + if hd != asker.Recorded { + return + } + e, _, found, ferr := h.byAsk(w.Ask) + if ferr != nil || !found || e.Stage != StageAsked { + return + } + if rerr := h.refuse(e, "the ask ended without an answer ("+string(w.Outcome)+")"); rerr != nil { + h.logf("%v", rerr) + } +} + +// Again asks the operator once more, when the router no longer holds the ask of an enrolment that is still at stage asked +// (ADR 0587 §8): a new ask under a new id, for the same node, address and expiry. +func (h *Holder) Again(node string) (IssueAnswer, error) { + row, found, err := h.Store.Get(NodeKey(node)) + if err != nil { + return IssueAnswer{}, err + } + if !found { + return IssueAnswer{}, fmt.Errorf("no enrolment of %s is open", node) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return IssueAnswer{}, err + } + if e.Stage != StageAsked { + return IssueAnswer{}, fmt.Errorf("the enrolment of %s is %s, not asked: there is nothing to ask again (a redeem stopped midway is finished by the machine calling redeem again)", node, e.Stage) + } + if !h.now().Before(e.Expires) { + return IssueAnswer{}, fmt.Errorf("the enrolment of %s has expired; revoke it and issue it again", node) + } + if h.Asker != nil { + if state, _, werr := h.whereIs(e.Ask); werr == nil && (state == "open" || state == "held") { + return IssueAnswer{}, fmt.Errorf("the ask %s is still %s at the router: the operator can answer it", e.Ask, state) + } + } + old := e.Ask + e.Ask = "enrol-" + node + "-" + h.now().UTC().Format("20060102t150405") + if e.Ask == old { + e.Ask += "b" + } + if _, err := h.Store.Update(NodeKey(node), e, row.Revision); err != nil { + return IssueAnswer{}, fmt.Errorf("the enrolment of %s moved meanwhile: %w", node, err) + } + if err := h.raise(e); err != nil { + return IssueAnswer{}, err + } + return IssueAnswer{Node: node, Address: e.Address, Ask: e.Ask, Expires: e.Expires, Says: "the operator is asked again"}, nil +} + +// whereIs is the router's state of an ask, when the asker can read it. +func (h *Holder) whereIs(id string) (string, *asks.Warrant, error) { + if w, ok := h.Asker.(interface { + Where(string) (string, *asks.Warrant, error) + }); ok { + return w.Where(id) + } + return "", nil, errors.New("the asker cannot read the router's record") +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go new file mode 100644 index 0000000..f94331a --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +var yes = asks.Option{ID: "yes"} + +// 774/S55/AC1: issue refuses, naming every finding, what it must: the lease with the controller, a bad name, a name a node +// record, a placement key or an open enrolment holds, and an expiry past a day. +func TestIssueRefusesNamingEveryFinding(t *testing.T) { + cases := []struct { + name string + setup func(r *rig) + node string + lasts time.Duration + want []string + }{ + {"the lease names the controller", func(r *rig) { r.rd["mesh-controller.lease"]["holder"] = `{"instance":"novox.mesh-controller"}` }, "ace", 0, + []string{"issuing is off until the controller's lease is written", "token issue"}}, + {"a bad name and a long expiry", nil, "Ace.1", 48 * time.Hour, []string{"not a valid node name", "outside zero to 24h0m0s"}}, + {"a placement key and a node record", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["ace.node"] = `{"v":1,"node":"ace","modules":[]}` + r.rd["node-engine.have"]["ace"] = `{}` + }, "ace", 0, []string{"ace has a placement key already"}}, + {"an open enrolment", func(r *rig) { r.seed("ace", "10.9.0.2", StageLive) }, "ace", 0, []string{"an enrolment of ace is open already"}}, + } + for _, c := range cases { + r := newRig(t) + if c.setup != nil { + c.setup(r) + } + _, err := r.h.Issue("person.operator", c.node, "a new machine", c.lasts) + if err == nil { + t.Fatalf("%s: issued", c.name) + } + for _, w := range c.want { + mustContain(t, err.Error(), w) + } + if len(r.asker.asked) != 0 || strings.Contains(strings.Join(r.keys(), " "), "address.10.9.0.3") { + t.Errorf("%s: asked %v, keys %v", c.name, r.asker.asked, r.keys()) + } + } +} + +// 774/S55/AC2: two issues at once for two nodes hold two addresses; a yes makes the token, moves both keys to live with the +// token's expiry, records the warrant, says enrolment-issued, and a second answer changes nothing. +func TestTwoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { + r := newRig(t) + var wg sync.WaitGroup + addrs := make([]string, 2) + for i, n := range []string{"ace", "g14"} { + wg.Add(1) + go func() { + defer wg.Done() + a, err := r.h.Issue("person.operator", n, "a new machine", 0) + if err != nil { + t.Error(err) + } + addrs[i] = a.Address + }() + } + wg.Wait() + if addrs[0] == "" || addrs[0] == addrs[1] || addrs[0] == "10.9.0.1" || addrs[1] == "10.9.0.1" { + t.Fatalf("addresses %v", addrs) + } + e, _ := r.enrolment("ace") + if e.Stage != StageAsked || e.Caller != "person.operator" || e.Ask == "" || len(r.asker.asked) != 2 { + t.Fatalf("%+v, asked %v", e, r.asker.asked) + } + ask, _ := r.h.AskOf(e) + if ask.Options[0].Level != asks.Approve || ask.Options[0].Binds == "" || ask.Check(r.now) != nil { + t.Errorf("the ask does not authorise the act: %+v, %v", ask.Options[0], ask.Check(r.now)) + } + if err := r.h.Warranted(yes, asks.Warrant{Ask: e.Ask}); err != nil { + t.Fatal(err) + } + live, _ := r.enrolment("ace") + if live.Stage != StageLive || r.st.ttl[NodeKey("ace")] != time.Hour || r.st.ttl[AddressKey(live.Address)] != time.Hour || + len(r.warr) != 1 || r.warr[0] != e.Ask || r.events()[0] != EventIssued { + t.Errorf("live %+v, ttl %v/%v, warrants %v, events %v", live, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey(live.Address)], r.warr, r.events()) + } + if strings.Contains(string(r.said[0].Body), "secret") || r.said[0].ID != eventID(EventIssued, live) { + t.Errorf("the event %s carries a secret-like word or no stable id", r.said[0].Body) + } + if err := r.h.Warranted(yes, asks.Warrant{Ask: e.Ask}); err != nil || len(r.said) != 1 { + t.Errorf("a second answer said %v: %v", r.events(), err) + } +} + +// 774/S55/AC3: a declined or expired ask, heard, takes both keys away, and enrolment-refused carries the key's content; a +// vault that fails leaves the enrolment at asked, for `again` to ask once more. +func TestADeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain(t *testing.T) { + r := newRig(t) + a, _ := r.h.Issue("person.operator", "ace", "a new machine", 0) + if err := r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: a.Ask}); err != nil { + t.Fatal(err) + } + if len(r.keys()) != 0 || r.events()[0] != EventRefused { + t.Fatalf("keys %v events %v", r.keys(), r.events()) + } + mustContain(t, string(r.said[0].Body), `"address":"`+a.Address+`"`) + b, _ := r.h.Issue("person.operator", "g14", "a new machine", 0) + r.h.Said(asker.Recorded, asks.Warrant{Ask: b.Ask, Outcome: asks.OutcomeExpired}, nil) + if len(r.keys()) != 0 || r.events()[1] != EventRefused { + t.Fatalf("keys %v events %v", r.keys(), r.events()) + } + c, _ := r.h.Issue("person.operator", "novox", "a new machine", 0) + r.vault.fail["token novox"] = true + if err := r.h.Warranted(yes, asks.Warrant{Ask: c.Ask}); err == nil { + t.Fatal("a vault that failed was taken for a token") + } + if e, _ := r.enrolment("novox"); e.Stage != StageAsked { + t.Fatalf("stage %s", e.Stage) + } + r.now = r.now.Add(time.Minute) + again, err := r.h.Again("novox") + if err != nil || again.Ask == c.Ask || len(r.asker.asked) != 4 { + t.Fatalf("again %+v, %v, asked %v", again, err, r.asker.asked) + } + if err := r.h.Warranted(yes, asks.Warrant{Ask: again.Ask}); err != nil { + t.Fatal(err) + } + if e, _ := r.enrolment("novox"); e.Stage != StageLive { + t.Errorf("stage %s", e.Stage) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index e907b6b..9107ca7 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -142,18 +142,26 @@ func (v *fakeVault) SignReply(node string, reply []byte) (json.RawMessage, error } type fakeAsker struct { + mu sync.Mutex asked, cancelled []string err error } func (a *fakeAsker) Ask(k asks.Ask) error { + a.mu.Lock() + defer a.mu.Unlock() if a.err != nil { return a.err } a.asked = append(a.asked, k.ID) return nil } -func (a *fakeAsker) Cancel(id string) error { a.cancelled = append(a.cancelled, id); return nil } +func (a *fakeAsker) Cancel(id string) error { + a.mu.Lock() + defer a.mu.Unlock() + a.cancelled = append(a.cancelled, id) + return nil +} type said struct { Event, ID string @@ -162,6 +170,7 @@ type said struct { // rig is a holder over fakes, with the lease handed to mesh-grants, one hub in placement and a clock the test moves. type rig struct { + mu sync.Mutex t *testing.T h *Holder st *memStore @@ -189,6 +198,8 @@ func newRig(t *testing.T) *rig { r.h = &Holder{Store: r.st, Read: r.rd, Vault: r.vault, Asker: r.asker, AskLasts: time.Hour, Say: func(event string, body any, id string) error { raw, _ := json.Marshal(body) + r.mu.Lock() + defer r.mu.Unlock() r.said = append(r.said, said{event, id, raw}) return nil }, diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go index 652bbc4..787a9d2 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "strings" "time" @@ -9,6 +10,8 @@ import ( // Tool names: the seat's verbs, as the manifest claims them (ADR 0587 §1, §8). const ( + IssueTool = Seat + ".issue" + AgainTool = Seat + ".again" TokensTool = Seat + ".tokens" ) @@ -31,11 +34,43 @@ func flag(a map[string]any, k string) bool { return a[k] == true || strings.EqualFold(arg(a, k), "true") } -var _ = time.Second +// lastsOf reads an expiry as a duration; none is the default. +func lastsOf(s string) (time.Duration, error) { + if s == "" { + return 0, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("the expiry %q is no duration (for example 1h): %w", s, err) + } + return d, nil +} // Tools are what the runtime serves. func Tools(h *Holder) []stdio.Tool { return []stdio.Tool{ + {Name: IssueTool, + Description: "Open an enrolment: let a new machine join the mesh (novox/hq ADR 0587 §4). Takes the node's name, a why and an " + + "expiry (one hour unless given, at most 24 hours). Refused at once, naming every finding, for a name that is no node name " + + "or that a node record, a placement key or an open enrolment holds, and for every call before the controller's lease is " + + "written to a holder that is no controller. Otherwise it chooses the lowest free private-network address, reserves it, " + + "writes the enrolment at stage asked and asks the operator on a channel that proves who answered; only their yes makes " + + "the token (the vault makes it, this seat never holds it). The caller is the one the runtime proves.", + Input: obj(map[string]any{"node": str("the new machine's name"), "why": str("why this machine joins"), + "expires": str("how long the token lasts, e.g. 1h (default) up to 24h")}, "node", "why"), + Run: func(a map[string]any) (any, error) { + lasts, err := lastsOf(arg(a, "expires")) + if err != nil { + return nil, err + } + return h.Issue(stdio.CallerOf(a), arg(a, "node"), arg(a, "why"), lasts) + }}, + {Name: AgainTool, + Description: "Ask the operator again about an enrolment still at stage asked whose ask the router no longer holds open " + + "(novox/hq ADR 0587 §8): a new ask for the same node, address and expiry. Nothing more: a redeem stopped midway is finished " + + "by the machine calling redeem again with the same keys.", + Input: obj(map[string]any{"node": str("the machine")}, "node"), + Run: func(a map[string]any) (any, error) { return h.Again(arg(a, "node")) }}, {Name: TokensTool, Description: "What the seat holds (novox/hq ADR 0587 §8): every open enrolment, or one node's, with its stage (asked, live or " + "redeeming), address, expiry, ask, revision and age; whether issuing is on and why (it is on only from the controller's " + From 89370b6f5c1e397db2465c4d3b3c95c5dd59a9ab Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:20:38 +0100 Subject: [PATCH 03/20] mesh-enrolment: redeem, the joining machine's one call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit redeem answers only enrol. as the runtime proves it, checks the proof against the identity key presented, spends the token once by compare-and-set (the keys lose their expiry), asks the vault for the node's account and certificate, writes the node into placement on the issue's warrant, answers once the bus node loaded node., signed by the vault; the same keys called again finish what is not done and answer the same (ADR 0587 §5-§7). The proof is mesh-host's EnrolProof with the node's name in the token's place, since the seat never holds the secret. Tests added: two table-style tests. (issue 774, 774/S56) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/kit_test.go | 8 +- .../cmd/mesh-enrolment/redeem.go | 166 ++++++++++++++++++ .../cmd/mesh-enrolment/redeem_test.go | 130 ++++++++++++++ .../cmd/mesh-enrolment/tools.go | 22 +++ 4 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 9107ca7..40bd2cc 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -203,7 +203,13 @@ func newRig(t *testing.T) *rig { r.said = append(r.said, said{event, id, raw}) return nil }, - AddNode: func(a AddNode) error { r.added = append(r.added, a); return r.placeErr }, + AddNode: func(a AddNode) error { + r.added = append(r.added, a) + if r.placeErr == nil { // placement writes the node as the assigner does on the warrant + r.rd["mesh-delivery-assigner.placement"][a.Node+".node"] = `{"v":1,"node":"` + a.Node + `","decided":{"address":"` + a.Address + `"},"modules":[]}` + } + return r.placeErr + }, Warrant: func(ask string) error { r.warr = append(r.warr, ask); return nil }, Act: func(what, why, cause string) error { r.acts = append(r.acts, what); return nil }, Loaded: func(string) (bool, error) { return r.loaded, nil }, diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go new file mode 100644 index 0000000..d714e4f --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -0,0 +1,166 @@ +package main + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "time" +) + +// RedeemArgs are what a joining node-engine says at redeem (ADR 0587 §6): the public halves of the keys it made where it +// runs, and a proof, signed with its identity key, that it holds the private half of the first. +type RedeemArgs struct { + Node string `json:"node"` + PublicKey []byte `json:"public_key"` + OverlayKey string `json:"overlay_key,omitempty"` + SealingKey string `json:"sealing_key,omitempty"` + ServingKey string `json:"serving_key,omitempty"` + Profile map[string]any `json:"profile,omitempty"` + Proof []byte `json:"proof"` +} + +// Keys are the public halves presented. +func (a RedeemArgs) Keys() NodeKeys { + return NodeKeys{Public: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey} +} + +// RedeemProof is the bytes a node signs with its identity key when it redeems: the node's name and every key it presents, so a +// proof cannot be moved to another request or another node. The layout is mesh-host's EnrolProof (internal/link/enrol.go), +// with the node's name in the place of the token's secret, which this seat never holds. +func RedeemProof(node string, k NodeKeys) []byte { + return []byte("novox-mesh-enrol\x00" + node + "\x00" + base64.StdEncoding.EncodeToString(k.Public) + + "\x00" + k.Overlay + "\x00" + k.Sealing + "\x00" + k.Serving) +} + +// RedeemReply is what the vault signs: the node's account sealed to its sealing key, the signed certificate for its internal +// name and the address the operator approved. +type RedeemReply struct { + Node string `json:"node"` + Address string `json:"address"` + Account json.RawMessage `json:"account"` + Certificate json.RawMessage `json:"certificate"` +} + +// RedeemAnswer is what redeem answers: the reply's bytes and the vault's signature over them. The node-engine refuses an +// answer not signed by the key its token carries. +type RedeemAnswer struct { + Reply []byte `json:"reply"` + Signed json.RawMessage `json:"signed"` +} + +// Redeem is the joining node-engine's one call (ADR 0587 §5, §6). It answers only a caller the runtime proves to be +// enrol., for a key at stage live (or at redeeming with the same keys' digest), checks the proof against the identity key +// presented, and then, each step read back and none repeated that is done: spends the token (the key to redeeming, by +// compare-and-set, with no expiry), asks the vault for the node's account and the certificate, writes the node into placement, +// waits for the bus node to load node., and answers the whole, signed by the vault. Called again with the same keys it +// finishes what is not done and answers the same. +func (h *Holder) Redeem(caller string, a RedeemArgs) (RedeemAnswer, error) { + if err := CheckNode(a.Node); err != nil { + return RedeemAnswer{}, err + } + if caller != EnrolUser(a.Node) { + return RedeemAnswer{}, fmt.Errorf("only %s may redeem the enrolment of %s, and the runtime proved the caller to be %q", EnrolUser(a.Node), a.Node, caller) + } + keys := a.Keys() + if len(a.PublicKey) != ed25519.PublicKeySize || !ed25519.Verify(a.PublicKey, RedeemProof(a.Node, keys), a.Proof) { + return RedeemAnswer{}, errors.New("the proof does not verify against the identity key presented: the enrolment is not spent") + } + digest := keys.Digest() + e, err := h.spend(a.Node, digest) + if err != nil { + return RedeemAnswer{}, err + } + account, err := h.Vault.Account(e.Node, a.SealingKey, digest) + if err != nil { + return RedeemAnswer{}, fmt.Errorf("the vault did not give the account of %s (call redeem again with the same keys): %w", e.Node, err) + } + cert, err := h.Vault.SignCertificate(e.Node, a.ServingKey) + if err != nil { + return RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) + } + if err := h.place(e, a, keys); err != nil { + return RedeemAnswer{}, err + } + if h.Loaded != nil { + loaded, err := h.Loaded(NodeUser(e.Node)) + if err != nil { + return RedeemAnswer{}, fmt.Errorf("whether the bus node loaded %s cannot be read (call redeem again with the same keys): %w", NodeUser(e.Node), err) + } + if !loaded { + return RedeemAnswer{}, fmt.Errorf("the bus node has not loaded %s yet (call redeem again with the same keys)", NodeUser(e.Node)) + } + } + reply, err := json.Marshal(RedeemReply{Node: e.Node, Address: e.Address, Account: account, Certificate: cert}) + if err != nil { + return RedeemAnswer{}, err + } + signed, err := h.Vault.SignReply(e.Node, reply) + if err != nil { + return RedeemAnswer{}, fmt.Errorf("the vault did not sign the answer (call redeem again with the same keys): %w", err) + } + return RedeemAnswer{Reply: reply, Signed: signed}, nil +} + +// spend moves the key from live to redeeming with the presented keys' digest, by compare-and-set, before its expiry; at +// redeeming it only checks the digest. A key that moved meanwhile is read again once. +func (h *Holder) spend(node, digest string) (Enrolment, error) { + for attempt := 0; attempt < 3; attempt++ { + row, found, err := h.Store.Get(NodeKey(node)) + if err != nil { + return Enrolment{}, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err) + } + if !found { + return Enrolment{}, fmt.Errorf("no enrolment of %s is open: it was never issued, has ended or has expired", node) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return Enrolment{}, err + } + switch e.Stage { + case StageRedeeming: + if e.KeysDigest != digest { + return Enrolment{}, fmt.Errorf("the enrolment of %s was redeemed with other keys: a token is spent once", node) + } + return e, nil + case StageAsked: + return Enrolment{}, fmt.Errorf("the enrolment of %s waits for the operator's yes: no token exists yet", node) + } + if !h.now().Before(e.Expires) { + return Enrolment{}, fmt.Errorf("the enrolment of %s expired at %s", node, e.Expires.UTC().Format(time.RFC3339)) + } + e.Stage, e.KeysDigest = StageRedeeming, digest + if _, err := h.Store.Update(NodeKey(node), e, row.Revision); err != nil { + if errors.Is(err, ErrChanged) { + continue + } + return Enrolment{}, fmt.Errorf("the enrolment of %s cannot be spent: %w", node, err) + } + // The move to redeeming rewrites both keys without an expiry, so a redeem under way never loses its key (§5). + if _, err := h.Store.Put(AddressKey(e.Address), Reservation{V: 1, Node: node}, 0); err != nil { + return Enrolment{}, fmt.Errorf("the address key %s cannot be kept (call redeem again with the same keys): %w", e.Address, err) + } + return e, nil + } + return Enrolment{}, fmt.Errorf("the enrolment of %s kept changing: call redeem again", node) +} + +// place writes the node into placement with the issue's warrant, unless placement holds it at the approved address already. +func (h *Holder) place(e Enrolment, a RedeemArgs, keys NodeKeys) error { + placements, err := h.placements() + if err != nil { + return fmt.Errorf("placement cannot be read (call redeem again with the same keys): %w", err) + } + if p, ok := placements[e.Node]; ok && p.Decided.Address == e.Address { + return nil + } + if h.AddNode == nil { + return errors.New("no route to mesh-placement") + } + if err := h.AddNode(AddNode{Node: e.Node, Address: e.Address, Why: "enrolment of " + e.Node + ": " + e.Why, Keys: keys, + Warrant: Warrant{Ask: e.Ask, Node: e.Node, Address: e.Address}}); err != nil { + return fmt.Errorf("%w (call redeem again with the same keys)", err) + } + return nil +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go new file mode 100644 index 0000000..0d3e2e5 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "testing" + "time" +) + +// engineEnrolProof is mesh-host's EnrolProof (internal/link/enrol.go:112-114), copied: the bytes a node-engine signs. The +// holder's RedeemProof must be the same bytes with the node's name in the token's place. +func engineEnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { + return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + + "\x00" + overlay + "\x00" + sealing + "\x00" + serving) +} + +// redeemArgs are a node's redeem call with a proof signed by the key seeded, over the keys given. +func redeemArgs(node string, seed byte, serving string) RedeemArgs { + priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{seed}, ed25519.SeedSize)) + a := RedeemArgs{Node: node, PublicKey: priv.Public().(ed25519.PublicKey), OverlayKey: "overlay-" + node, SealingKey: "sealing-" + node, ServingKey: serving} + a.Proof = ed25519.Sign(priv, RedeemProof(node, a.Keys())) + return a +} + +// 774/S56/AC1: a redeem from another caller, with a bad proof, after the expiry, or with other keys than the first, is +// refused (one row each), and a good one spends the token, writes the node into placement on the issue's warrant and +// answers once the bus node loaded the account. +func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { + good := redeemArgs("ace", 1, "serving-a") + bad := good + bad.Proof = append([]byte{}, good.Proof...) + bad.Proof[0] ^= 1 + if !bytes.Equal(RedeemProof("ace", good.Keys()), engineEnrolProof("ace", good.PublicKey, good.OverlayKey, good.SealingKey, good.ServingKey)) { + t.Fatal("the proof's bytes are not the engine's") + } + cases := []struct { + name string + caller string + args RedeemArgs + setup func(r *rig) + want string + }{ + {"another caller", "enrol.g14", good, nil, "only enrol.ace may redeem"}, + {"a bad proof", "enrol.ace", bad, nil, "proof does not verify"}, + {"after the expiry", "enrol.ace", good, func(r *rig) { r.now = r.now.Add(2 * time.Hour) }, "expired at"}, + {"other keys than the first", "enrol.ace", redeemArgs("ace", 2, "serving-b"), func(r *rig) { + e, _ := r.enrolment("ace") + e.Stage, e.KeysDigest = StageRedeeming, good.Keys().Digest() + _, _ = r.st.Put(NodeKey("ace"), e, 0) + }, "spent once"}, + {"before the operator's yes", "enrol.ace", good, func(r *rig) { + e, _ := r.enrolment("ace") + e.Stage = StageAsked + _, _ = r.st.Put(NodeKey("ace"), e, 0) + }, "waits for the operator's yes"}, + {"no enrolment", "enrol.novox", redeemArgs("novox", 3, "s"), nil, "no enrolment of novox is open"}, + } + for _, c := range cases { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + if c.setup != nil { + c.setup(r) + } + _, err := r.h.Redeem(c.caller, c.args) + if err == nil { + t.Errorf("%s: redeemed", c.name) + continue + } + mustContain(t, err.Error(), c.want) + if len(r.vault.calls) != 0 || len(r.added) != 0 { + t.Errorf("%s: the vault %v or placement %v was called", c.name, r.vault.calls, r.added) + } + } + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")] = time.Hour, time.Hour + ans, err := r.h.Redeem("enrol.ace", good) + if err != nil { + t.Fatal(err) + } + e, _ := r.enrolment("ace") + if e.Stage != StageRedeeming || e.KeysDigest != good.Keys().Digest() || r.st.ttl[NodeKey("ace")] != 0 || r.st.ttl[AddressKey("10.9.0.2")] != 0 { + t.Errorf("spent %+v, ttl %v/%v", e, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")]) + } + if len(r.added) != 1 || r.added[0].Warrant != (Warrant{Ask: "ask-ace", Node: "ace", Address: "10.9.0.2"}) || len(ans.Reply) == 0 || len(ans.Signed) == 0 { + t.Errorf("placement %+v, answer %+v", r.added, ans) + } + r.loaded = false + if _, err := r.h.Redeem("enrol.ace", good); err == nil { + t.Error("answered before the bus node loaded the account") + } +} + +// 774/S56/AC2: a redeem stopped after the vault's call, or after the placement call, called again with the same keys, +// finishes and answers the same, and the vault minted once (it is asked for the same account). +func TestARedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { + good := redeemArgs("ace", 1, "serving-a") + whole := newRig(t) + whole.seed("ace", "10.9.0.2", StageLive) + want, err := whole.h.Redeem("enrol.ace", good) + if err != nil { + t.Fatal(err) + } + for _, stop := range []string{"certificate ace", "sign ace"} { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + r.vault.fail[stop] = true + if _, err := r.h.Redeem("enrol.ace", good); err == nil { + t.Fatalf("%s: no error", stop) + } + got, err := r.h.Redeem("enrol.ace", good) + if err != nil || !bytes.Equal(got.Reply, want.Reply) || string(got.Signed) != string(want.Signed) { + t.Errorf("%s: %v, answer %+v, want %+v", stop, err, got, want) + } + if stop == "sign ace" && len(r.added) != 1 { + t.Errorf("placement was written %d times", len(r.added)) + } + } + // Placement that already holds the node at the approved address is not written again. + r := newRig(t) + r.seed("ace", "10.9.0.2", StageRedeeming) + e, _ := r.enrolment("ace") + e.KeysDigest = good.Keys().Digest() + _, _ = r.st.Put(NodeKey("ace"), e, 0) + r.rd["mesh-delivery-assigner.placement"]["ace.node"] = `{"v":1,"node":"ace","decided":{"address":"10.9.0.2"},"modules":[]}` + if _, err := r.h.Redeem("enrol.ace", good); err != nil || len(r.added) != 0 { + t.Errorf("%v, placement written %d times", err, len(r.added)) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go index 787a9d2..3777b71 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "strings" "time" @@ -12,6 +13,7 @@ import ( const ( IssueTool = Seat + ".issue" AgainTool = Seat + ".again" + RedeemTool = Seat + ".redeem" TokensTool = Seat + ".tokens" ) @@ -65,6 +67,26 @@ func Tools(h *Holder) []stdio.Tool { } return h.Issue(stdio.CallerOf(a), arg(a, "node"), arg(a, "why"), lasts) }}, + {Name: RedeemTool, + Description: "The joining machine's one call (novox/hq ADR 0587 §5, §6), answered only for a caller the runtime proves to be " + + "enrol.: its node-engine presents the public halves of the keys it made where it runs and a proof signed with its " + + "identity key. The token is spent (once, with those keys), the vault gives the node's account and signs its certificate, " + + "the node is written into placement on the issue's warrant, and the answer is given once the bus node loaded node., " + + "signed by the vault. Called again with the same keys it finishes what is not done and answers the same.", + Input: obj(map[string]any{"node": str("the machine's name"), "public_key": str("its identity key, base64"), + "overlay_key": str("its private-network key"), "sealing_key": str("its sealing key, base64"), + "serving_key": str("its serving key"), "proof": str("the proof, base64")}, "node", "public_key", "proof"), + Run: func(a map[string]any) (any, error) { + var args RedeemArgs + raw, err := json.Marshal(a) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &args); err != nil { + return nil, fmt.Errorf("the redeem call does not read: %w", err) + } + return h.Redeem(stdio.CallerOf(a), args) + }}, {Name: AgainTool, Description: "Ask the operator again about an enrolment still at stage asked whose ask the router no longer holds open " + "(novox/hq ADR 0587 §8): a new ask for the same node, address and expiry. Nothing more: a redeem stopped midway is finished " + From f904ff425e4598015b21a81cac12f813fa479094 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:21:18 +0100 Subject: [PATCH 04/20] mesh-enrolment: confirm closes an enrolment on the node's own call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the node dialled as node. confirms (the runtime's proof of the caller is the confirmation, ADR 0587 §6); the holder then says node-enrolled with the key's content and deletes the address key (while it still names the node) and the node's key. A stop between the event and the deletes is finished by calling confirm again; the event keeps one message. release now answers what it could not delete. Tests added: one table-style test. (issue 774, 774/S57) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/confirm.go | 42 ++++++++++++++++++ .../cmd/mesh-enrolment/confirm_test.go | 44 +++++++++++++++++++ .../cmd/mesh-enrolment/issue.go | 35 +++++++++++---- .../cmd/mesh-enrolment/tools.go | 15 +++++-- 4 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go new file mode 100644 index 0000000..ee8977a --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + "strings" +) + +// Confirm ends an enrolment that came through (ADR 0587 §6): the node, dialled as its own login node. after it +// redeemed, calls it, and the runtime's proof of that caller is the confirmation (as ADR 0583 §3). Only then does the holder +// say node-enrolled, with the key's whole content, and delete the key and its address key. The node's name is the caller's +// own; an argument naming another node is refused. A confirm called again after a stop between the event and the deletes +// finishes the deletes and says nothing twice (the bus keeps one message of one id). +func (h *Holder) Confirm(caller, node string) (map[string]any, error) { + own, ok := strings.CutPrefix(caller, nodePrefix) + if !ok || CheckNode(own) != nil || strings.Contains(own, "--") { + return nil, fmt.Errorf("only the node itself, dialled as node., confirms its enrolment, and the runtime proved the caller to be %q", caller) + } + if node != "" && node != own { + return nil, fmt.Errorf("%s confirms its own enrolment only, not that of %s", caller, node) + } + row, found, err := h.Store.Get(NodeKey(own)) + if err != nil { + return nil, fmt.Errorf("the enrolment of %s cannot be read: %w", own, err) + } + if !found { + return nil, fmt.Errorf("no enrolment of %s is open: nothing to confirm", own) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return nil, err + } + if e.Stage != StageRedeeming { + return nil, fmt.Errorf("the enrolment of %s is %s: it is confirmed after redeem", own, e.Stage) + } + if err := h.sayEvent(EventEnrolled, e, ""); err != nil { + return nil, fmt.Errorf("node-enrolled of %s cannot be said, so nothing is deleted (call confirm again): %w", own, err) + } + if err := h.release(e); err != nil { + return nil, fmt.Errorf("the enrolment of %s is confirmed and not closed (call confirm again): %w", own, err) + } + return map[string]any{"node": own, "address": e.Address, "says": "the node is enrolled; its enrolment is closed and the vault is asked to drop the token"}, nil +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go new file mode 100644 index 0000000..b88928c --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go @@ -0,0 +1,44 @@ +package main + +import "testing" + +// 774/S57/AC1: a confirm from any caller but node. is refused; from node. node-enrolled is said and both keys +// are deleted; a stop between the event and the deletes is finished by calling confirm again. +func TestConfirmIsTheNodesOwnAndEndsTheEnrolment(t *testing.T) { + for _, c := range []struct{ name, caller, node, want string }{ + {"the enrolment user", "enrol.ace", "", "dialled as node."}, + {"another node", "node.g14", "ace", "its own enrolment only"}, + {"the second login", "node.ace--b", "", "dialled as node."}, + {"no one", "unproven", "ace", "dialled as node."}, + } { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageRedeeming) + if _, err := r.h.Confirm(c.caller, c.node); err == nil { + t.Errorf("%s: confirmed", c.name) + } else { + mustContain(t, err.Error(), c.want) + } + if len(r.said) != 0 || len(r.keys()) != 2 { + t.Errorf("%s: said %v, keys %v", c.name, r.events(), r.keys()) + } + } + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + if _, err := r.h.Confirm("node.ace", ""); err == nil { + t.Error("confirmed before redeem") + } + r.seed("ace", "10.9.0.2", StageRedeeming) + r.st.failDelete[NodeKey("ace")] = true + if _, err := r.h.Confirm("node.ace", ""); err == nil { + t.Fatal("a delete that failed was taken for done") + } + if r.has(AddressKey("10.9.0.2")) || !r.has(NodeKey("ace")) || len(r.said) != 1 { + t.Fatalf("keys %v, said %v", r.keys(), r.events()) + } + if _, err := r.h.Confirm("node.ace", ""); err != nil || len(r.keys()) != 0 { + t.Fatalf("%v, keys %v", err, r.keys()) + } + if r.events()[0] != EventEnrolled || r.said[0].ID != r.said[1].ID { + t.Errorf("events %v ids %s %s", r.events(), r.said[0].ID, r.said[1].ID) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index e6f6be2..e701e00 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "errors" "fmt" "strconv" @@ -119,21 +120,38 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw return IssueAnswer{}, fmt.Errorf("the enrolment of %s cannot be written: %w", node, err) } if err := h.raise(e); err != nil { - h.release(e) + _ = h.release(e) return IssueAnswer{}, err } return IssueAnswer{Node: node, Address: e.Address, Ask: e.Ask, Expires: e.Expires, Says: "the operator is asked on their channel; only their yes makes the token. mesh-enrolment.tokens shows where it stands"}, nil } -// release takes an enrolment's two keys away, the address key last (the node's key is what names the reservation's owner). -func (h *Holder) release(e Enrolment) { - if err := h.Store.Delete(NodeKey(e.Node)); err != nil { - h.logf("the key of %s could not be deleted and stays: %v", e.Node, err) +// release takes an enrolment's two keys away, the address key first and only while it still names this node (a key some +// other enrolment now holds is never deleted), then the node's key, whose presence is what lets a call made again finish a +// release that stopped. It answers what could not be deleted. +func (h *Holder) release(e Enrolment) error { + var errs []error + if row, found, err := h.Store.Get(AddressKey(e.Address)); err != nil { + errs = append(errs, fmt.Errorf("the address key %s cannot be read: %w", e.Address, err)) + } else if found { + var r Reservation + if json.Unmarshal(row.Value, &r) == nil && r.Node == e.Node { + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + errs = append(errs, fmt.Errorf("the address key %s could not be deleted: %w", e.Address, err)) + } + } } - if err := h.Store.Delete(AddressKey(e.Address)); err != nil { - h.logf("the address key %s could not be deleted and stays: %v", e.Address, err) + if len(errs) == 0 { + if err := h.Store.Delete(NodeKey(e.Node)); err != nil { + errs = append(errs, fmt.Errorf("the key of %s could not be deleted: %w", e.Node, err)) + } } + if err := errors.Join(errs...); err != nil { + h.logf("%v", err) + return err + } + return nil } // AskOf is the ask of an enrolment, in the words the operator reads: that a machine of this name may join until the @@ -235,8 +253,7 @@ func (h *Holder) refuse(e Enrolment, reason string) error { if err := h.sayEvent(EventRefused, e, reason); err != nil { return fmt.Errorf("enrolment-refused of %s cannot be said, so its keys stay: %w", e.Node, err) } - h.release(e) - return nil + return h.release(e) } // Said is the asker's note of an ask that ended with no yes or no (expired, or refused by the router): the enrolment ends. diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go index 3777b71..e6f1ac3 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -11,10 +11,11 @@ import ( // Tool names: the seat's verbs, as the manifest claims them (ADR 0587 §1, §8). const ( - IssueTool = Seat + ".issue" - AgainTool = Seat + ".again" - RedeemTool = Seat + ".redeem" - TokensTool = Seat + ".tokens" + IssueTool = Seat + ".issue" + AgainTool = Seat + ".again" + RedeemTool = Seat + ".redeem" + ConfirmTool = Seat + ".confirm" + TokensTool = Seat + ".tokens" ) func obj(props map[string]any, required ...string) map[string]any { @@ -87,6 +88,12 @@ func Tools(h *Holder) []stdio.Tool { } return h.Redeem(stdio.CallerOf(a), args) }}, + {Name: ConfirmTool, + Description: "The new machine's confirmation (novox/hq ADR 0587 §6), answered only for the node dialled as its own login " + + "node. after it redeemed: the runtime's proof of that caller is the confirmation. Says node-enrolled and closes " + + "the enrolment; the vault is then asked to drop the token.", + Input: obj(map[string]any{"node": str("the machine's own name (optional)")}), + Run: func(a map[string]any) (any, error) { return h.Confirm(stdio.CallerOf(a), arg(a, "node")) }}, {Name: AgainTool, Description: "Ask the operator again about an enrolment still at stage asked whose ask the router no longer holds open " + "(novox/hq ADR 0587 §8): a new ask for the same node, address and expiry. Nothing more: a redeem stopped midway is finished " + From 02214cf633532e8935c991b8818e660981524e48 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:22:23 +0100 Subject: [PATCH 05/20] mesh-enrolment: revoke, the expiry marker and the drop of the token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit revoke ends an enrolment at any stage without an Approve and records it as an act done by hand (the vault retires an account minted for a redeem under way); the bus's expiry marker of a key at live says enrolment-expired with the issued event; the durable consumer of the seat's own ending events asks the vault to drop the token and its hash, acknowledging only once the vault answers, never before an end. No event or key holds a token (ADR 0587 §5, §8). Run starts the watch, the consumer and the operator's answers. Tests added: two table-style tests. (issue 774, 774/S58) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../mesh-enrolment/cmd/mesh-enrolment/end.go | 119 ++++++++++++++++++ .../cmd/mesh-enrolment/end_test.go | 91 ++++++++++++++ .../cmd/mesh-enrolment/kit_test.go | 2 + .../mesh-enrolment/cmd/mesh-enrolment/main.go | 2 + .../mesh-enrolment/cmd/mesh-enrolment/run.go | 39 ++++++ .../cmd/mesh-enrolment/tokens_test.go | 9 ++ .../cmd/mesh-enrolment/tools.go | 11 ++ 7 files changed, 273 insertions(+) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/end.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/run.go diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go new file mode 100644 index 0000000..b4d2f2b --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -0,0 +1,119 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Revoke ends an enrolment at any stage (ADR 0587 §5), by a person's call: at asked it withdraws the ask; at redeeming it also +// undoes what redeem had not finished, the vault retiring a node account it already minted (a node already written into +// placement stays there and is removed, if wanted, as any node is removed). It only takes access away, so it needs no Approve; +// it is recorded in the hand-act log. enrolment-revoked carries the key's content and the why; the token is then dropped by +// the durable consumer of that event, never before. A revoke that stopped is finished by calling it again. +func (h *Holder) Revoke(caller, node, why string) (map[string]any, error) { + if caller == "" || caller == stdio.Unproven { + return nil, fmt.Errorf("the caller is not proven by the runtime, and a revoke is recorded with who made it") + } + if strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("a revoke says why: it is recorded as an act done by hand") + } + row, found, err := h.Store.Get(NodeKey(node)) + if err != nil { + return nil, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err) + } + if !found { + return nil, fmt.Errorf("no enrolment of %s is open: mesh-enrolment.tokens lists those that are", node) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return nil, err + } + if e.Stage == StageAsked && e.Ask != "" && h.Asker != nil { + if err := h.Asker.Cancel(e.Ask); err != nil { + h.logf("the ask %s of %s was not taken back at the router: %v", e.Ask, node, err) + } + } + if e.Stage == StageRedeeming { + if err := h.Vault.Retire(node); err != nil { + return nil, fmt.Errorf("the vault did not retire the account minted for %s, so nothing is ended (call revoke again): %w", node, err) + } + } + out := map[string]any{"node": node, "was": e.Stage} + if h.Act != nil { + if err := h.Act("mesh-enrolment revoke of "+node+" at "+string(e.Stage), why+" (by "+caller+")", "enrolment-revoke"); err != nil { + out["recorded"] = "NOT recorded in the hand-act log: " + err.Error() + h.logf("the revoke of %s is not recorded in the hand-act log: %v", node, err) + } + } + if err := h.sayEvent(EventRevoked, e, why); err != nil { + return nil, fmt.Errorf("enrolment-revoked of %s cannot be said, so its keys stay (call revoke again): %w", node, err) + } + if err := h.release(e); err != nil { + return nil, fmt.Errorf("the enrolment of %s is revoked and its keys are not all deleted (call revoke again): %w", node, err) + } + out["says"] = "the enrolment is ended; the vault is asked to drop the token and, for a redeem under way, the account was retired" + return out, nil +} + +// Heard is the holder's watch of its own bucket (stdio.State.Follow): it keeps each node key's last content, and the bus's +// expiry marker of a key at stage live says enrolment-expired, naming the node and the enrolment-issued event that carried its +// content (ADR 0587 §5). redeem checks the expiry itself, so a marker heard late admits no machine. +func (h *Holder) Heard(op, key string, value []byte) error { + node, ok := strings.CutPrefix(key, "node.") + if !ok { + return nil + } + h.mu.Lock() + defer h.mu.Unlock() + if h.seen == nil { + h.seen = map[string]Enrolment{} + } + switch op { + case stdio.OpPut: + if e, err := decodeEnrolment(value); err == nil { + h.seen[node] = e + } + case stdio.OpDelete: + delete(h.seen, node) + case stdio.OpExpire: + e, known := h.seen[node] + delete(h.seen, node) + body := Event{Node: node, Enrolment: e} + id := "enrolment-expired." + node + if known { + body.Issued = eventID(EventIssued, e) + id = eventID(EventExpired, e) + } + body.Reason = "the token's expiry passed unredeemed" + if err := h.say(EventExpired, body, id); err != nil { + return fmt.Errorf("enrolment-expired of %s cannot be said (offered again): %w", node, err) + } + } + return nil +} + +// Ended is the durable consumer's act on one of the holder's own ending events (ADR 0587 §5): the vault is asked to drop the +// node's token and its hash, and the event is acknowledged only when the vault answers that both are gone, so a drop that +// fails is delivered again by the bus. It never runs before an end: the consumer hears only the end events. +func (h *Holder) Ended(event string, body []byte) error { + var e struct { + Node string `json:"node"` + } + if err := json.Unmarshal(body, &e); err != nil || CheckNode(e.Node) != nil { + h.logf("the event %s names no node and is not acted on: %v", event, err) + return nil + } + if err := h.Vault.Drop(e.Node); err != nil { + return fmt.Errorf("the vault did not drop the token of %s (the bus delivers %s again): %w", e.Node, event, err) + } + h.mu.Lock() + h.ended = append(h.ended, json.RawMessage(`{"event":`+fmt.Sprintf("%q", event)+`,"content":`+string(body)+`}`)) + if len(h.ended) > 200 { + h.ended = h.ended[len(h.ended)-200:] + } + h.mu.Unlock() + return nil +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go new file mode 100644 index 0000000..92cd90f --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// 774/S58/AC1: a revoke at redeeming, whether the vault minted the account yet or not, takes both keys away, the vault retires +// the account, the revoke is recorded and enrolment-revoked is said; one the vault failed to retire ends nothing and is +// finished by calling it again. At asked it withdraws the ask; an expiry marker says enrolment-expired with the issued event. +func TestRevokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageRedeeming) + r.vault.fail["retire ace"] = true + if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err == nil || len(r.keys()) != 2 || len(r.said) != 0 { + t.Fatalf("a failed retire ended it: %v, keys %v, events %v", err, r.keys(), r.events()) + } + if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err != nil { + t.Fatal(err) + } + if len(r.keys()) != 0 || !slices.Equal(r.events(), []string{EventRevoked}) || len(r.acts) != 1 || !slices.Contains(r.vault.calls, "retire ace") { + t.Errorf("keys %v events %v acts %v vault %v", r.keys(), r.events(), r.acts, r.vault.calls) + } + mustContain(t, string(r.said[0].Body), "never confirmed") + // At asked it withdraws the ask and the vault is not asked to retire anything. + r.seed("g14", "10.9.0.3", StageAsked) + if _, err := r.h.Revoke("person.operator", "g14", "changed our mind"); err != nil || !slices.Equal(r.asker.cancelled, []string{"ask-g14"}) || slices.Contains(r.vault.calls, "retire g14") { + t.Errorf("%v, cancelled %v, vault %v", err, r.asker.cancelled, r.vault.calls) + } + if _, err := r.h.Revoke(stdio.Unproven, "ace", "x"); err == nil { + t.Error("revoked for no one") + } + // The bus's marker of a key at live: the content the watch handed over is said with the issued event. + r.seed("novox", "10.9.0.4", StageLive) + row, _, _ := r.st.Get(NodeKey("novox")) + _ = r.h.Heard(stdio.OpPut, NodeKey("novox"), row.Value) + if err := r.h.Heard(stdio.OpExpire, NodeKey("novox"), nil); err != nil { + t.Fatal(err) + } + last := r.said[len(r.said)-1] + mustContain(t, string(last.Body), `"issued-event":"enrolment-issued.novox.`) + if last.Event != EventExpired { + t.Errorf("said %s", last.Event) + } +} + +// 774/S58/AC2 and AC3: the drop is asked of the vault only from an ending event, acknowledged only once the vault answers, and +// asked again when its first answer fails; and neither an event nor the bucket ever holds the token, sealed or not. +func TestTheDropFollowsOnlyAnEndAndNothingHoldsTheToken(t *testing.T) { + r := newRig(t) + a, _ := r.h.Issue("person.operator", "ace", "a new machine", 0) + _ = r.h.Warranted(yes, askWarrant(a.Ask)) + if _, err := r.h.Tokens("ace", false); err != nil { // relays the sealed token + t.Fatal(err) + } + if slices.ContainsFunc(r.vault.calls, func(c string) bool { return strings.HasPrefix(c, "drop") }) { + t.Fatalf("the vault was asked to drop a token before an end: %v", r.vault.calls) + } + if _, err := r.h.Revoke("person.operator", "ace", "lost the machine"); err != nil { + t.Fatal(err) + } + end := r.said[len(r.said)-1] + r.vault.fail["drop ace"] = true + if err := r.h.Ended(end.Event, end.Body); err == nil { + t.Fatal("an event was acknowledged though the vault did not answer") + } + if err := r.h.Ended(end.Event, end.Body); err != nil { + t.Fatal(err) + } + if n := strings.Count(strings.Join(r.vault.calls, ","), "drop ace"); n != 2 { + t.Errorf("the drop was asked %d times", n) + } + for _, s := range r.said { + if strings.Contains(string(s.Body), "sealed") || strings.Contains(string(s.Body), "token") { + t.Errorf("the event %s holds a token: %s", s.Event, s.Body) + } + } + for k, row := range r.st.rows { + if strings.Contains(string(row.Value), "sealed") { + t.Errorf("the key %s holds a token", k) + } + } + tk, _ := r.h.Tokens("", true) + if raw, _ := json.Marshal(tk); !strings.Contains(string(raw), "enrolment-revoked") { + t.Errorf("history %s", raw) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 40bd2cc..d42413b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -255,3 +255,5 @@ func errText(err error) string { } return fmt.Sprint(err) } + +func askWarrant(ask string) asks.Warrant { return asks.Warrant{Ask: ask} } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go index 189f172..3ff7ccd 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -139,6 +139,8 @@ func loadedVia(rd Reader) func(string) (bool, error) { func main() { h := newHolder() + client := h.Asker.(*asker.Client) + h.Run(func() error { return client.Listen(h.Warranted, h.Said) }) if err := stdio.Serve("", Tools(h)); err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/run.go b/modules/mesh-enrolment/cmd/mesh-enrolment/run.go new file mode 100644 index 0000000..07cd094 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/run.go @@ -0,0 +1,39 @@ +package main + +import ( + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Run starts what the holder does beside its tools, none of it on a timer: the watch of its own bucket (the bus's expiry +// markers), the durable consumer of its own ending events (the drop), and the operator's answers. Each is asked of the +// runtime as soon as it serves, and again with a growing wait until the runtime takes it. +func (h *Holder) Run(listen func() error) { + go stdio.State(State).Follow("", func(c stdio.StateChange) error { return h.Heard(c.Op, c.Key, c.Value) }) + for _, event := range EndEvents { + go func(key string) { + <-stdio.Served() + retry("hear "+key, h.logf, func() error { + return stdio.Subscribe(key, func(e stdio.Envelope) error { return h.Ended(key, e.Body) }) + }) + }(Seat + "." + event) + } + if listen != nil { + go func() { + <-stdio.Served() + retry("hear the operator's answers", h.logf, listen) + }() + } +} + +func retry(what string, logf func(string, ...any), do func() error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := do() + if err == nil { + return + } + logf("does not %s yet (%v); asking again in %s", what, err, wait) + time.Sleep(wait) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index a200701..0ca284c 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -94,6 +94,15 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if !slices.Equal(m.Consumes, heard) { t.Errorf("consumes %v, the end events are %v", m.Consumes, heard) } + served := map[string]bool{} + for _, tool := range Tools(&Holder{}) { + served[tool.Name] = true + } + for _, v := range Verbs { + if !served[Seat+"."+v] { + t.Errorf("the verb %s has no tool", v) + } + } if len(m.State) != 2 || string(m.State[0]) == "" || !slices.ContainsFunc(m.State, func(s json.RawMessage) bool { var d struct { Name string `json:"name"` diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go index e6f1ac3..1c1ca47 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -15,6 +15,7 @@ const ( AgainTool = Seat + ".again" RedeemTool = Seat + ".redeem" ConfirmTool = Seat + ".confirm" + RevokeTool = Seat + ".revoke" TokensTool = Seat + ".tokens" ) @@ -94,6 +95,16 @@ func Tools(h *Holder) []stdio.Tool { "the enrolment; the vault is then asked to drop the token.", Input: obj(map[string]any{"node": str("the machine's own name (optional)")}), Run: func(a map[string]any) (any, error) { return h.Confirm(stdio.CallerOf(a), arg(a, "node")) }}, + {Name: RevokeTool, + Description: "End an enrolment at any stage (novox/hq ADR 0587 §5): at asked it withdraws the ask; at live or redeeming it deletes " + + "the key and its address key, and at redeeming the vault retires an account it already minted (a node already written into " + + "placement stays there). It only takes access away, so it needs no Approve; it is recorded as an act done by hand. Says " + + "enrolment-revoked; the vault then drops the token. A machine stuck at redeeming is ended by this. A revoke that stopped is " + + "finished by calling it again.", + Input: obj(map[string]any{"node": str("the machine"), "why": str("why, kept with the act")}, "node", "why"), + Run: func(a map[string]any) (any, error) { + return h.Revoke(stdio.CallerOf(a), arg(a, "node"), arg(a, "why")) + }}, {Name: AgainTool, Description: "Ask the operator again about an enrolment still at stage asked whose ask the router no longer holds open " + "(novox/hq ADR 0587 §8): a new ask for the same node, address and expiry. Nothing more: a redeem stopped midway is finished " + From 07d26adcd3637ea97c9597ddf4fde68b7ac38fb7 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:25:02 +0100 Subject: [PATCH 06/20] mesh-enrolment: read the group's mesh-sdk contract instead of its own copies of the shapes The seat, the enrolment state, redeem and its proof, the answer the vault signs, and the vault's two verbs' answers are now mesh-sdk's (catalogue and delivery), which the vault, the grants part and the engine read too; the proof is the contract's RedeemProof, the answer carries the login, account, certificate and the vault's signing key (read from its state signers), and the bucket's values are written through the contract's codecs. The expiry event names the issued event by message id as well as by the sequence, which a bundle cannot learn. The module builds against the group's mesh-sdk head through a go.work outside the tree; the lead re-pins go.mod before the push. Tests merged into the same functions; none added. (issue 774, 774/S54-S58) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../mesh-enrolment/cmd/mesh-enrolment/end.go | 14 ++- .../cmd/mesh-enrolment/end_test.go | 2 +- .../cmd/mesh-enrolment/enrolment.go | 95 +++++++----------- .../cmd/mesh-enrolment/holder.go | 35 +++---- .../cmd/mesh-enrolment/issue.go | 13 +-- .../cmd/mesh-enrolment/kit_test.go | 27 +++--- .../mesh-enrolment/cmd/mesh-enrolment/main.go | 27 ++++-- .../cmd/mesh-enrolment/redeem.go | 97 +++++++------------ .../cmd/mesh-enrolment/redeem_test.go | 36 +++---- .../cmd/mesh-enrolment/tokens.go | 24 ++--- .../cmd/mesh-enrolment/tokens_test.go | 14 +-- .../cmd/mesh-enrolment/tools.go | 16 +-- .../cmd/mesh-enrolment/vault.go | 75 ++++++++++---- modules/mesh-enrolment/module.json | 1 + 14 files changed, 239 insertions(+), 237 deletions(-) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go index b4d2f2b..79e75c2 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -6,6 +6,7 @@ import ( "strings" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Revoke ends an enrolment at any stage (ADR 0587 §5), by a person's call: at asked it withdraws the ask; at redeeming it also @@ -81,13 +82,13 @@ func (h *Holder) Heard(op, key string, value []byte) error { case stdio.OpExpire: e, known := h.seen[node] delete(h.seen, node) - body := Event{Node: node, Enrolment: e} id := "enrolment-expired." + node if known { - body.Issued = eventID(EventIssued, e) id = eventID(EventExpired, e) } - body.Reason = "the token's expiry passed unredeemed" + // The sequence of the enrolment-issued event on the events' stream is not given to a bundle that publishes (the runtime + // answers no sequence), so issued-event is 0 and issued-id names the message id that carried the content. + body := Expired{EnrolmentExpiry: delivery.EnrolmentExpiry{V: delivery.Version, Node: node}, IssuedID: eventID(EventIssued, e)} if err := h.say(EventExpired, body, id); err != nil { return fmt.Errorf("enrolment-expired of %s cannot be said (offered again): %w", node, err) } @@ -95,6 +96,13 @@ func (h *Holder) Heard(op, key string, value []byte) error { return nil } +// Expired is what enrolment-expired carries (delivery.EnrolmentExpiry) and the message id of the enrolment-issued event that +// carried the key's content, which the bus's sequence number cannot be known for. +type Expired struct { + delivery.EnrolmentExpiry + IssuedID string `json:"issued-id,omitempty"` +} + // Ended is the durable consumer's act on one of the holder's own ending events (ADR 0587 §5): the vault is asked to drop the // node's token and its hash, and the event is acknowledged only when the vault answers that both are gone, so a drop that // fails is delivered again by the bus. It never runs before an end: the consumer hears only the end events. diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go index 92cd90f..9bde32a 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -42,7 +42,7 @@ func TestRevokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { t.Fatal(err) } last := r.said[len(r.said)-1] - mustContain(t, string(last.Body), `"issued-event":"enrolment-issued.novox.`) + mustContain(t, string(last.Body), `"issued-id":"enrolment-issued.novox.`) if last.Event != EventExpired { t.Errorf("said %s", last.Event) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go index f23eaae..f492b5d 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go @@ -6,6 +6,9 @@ import ( "fmt" "regexp" "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // mesh-enrolment: the holder of the seat mesh-enrolment (novox/hq ADR 0587 §1), held once for the mesh. A Go bundle served @@ -15,75 +18,54 @@ import ( // Module and Seat are the module's name and the seat it holds. const ( - Module = "mesh-enrolment" - Seat = "mesh-enrolment" + Module = catalogue.EnrolmentSeat + Seat = catalogue.EnrolmentSeat // State is the holder's bucket of open enrolments; its keys are node. and address.
(ADR 0587 §2). - State = "enrolments" + State = delivery.EnrolmentsState ) // The seat's six verbs and five events (ADR 0587 §1). -var Verbs = []string{"issue", "redeem", "confirm", "revoke", "tokens", "again"} +var Verbs = []string{catalogue.EnrolmentVerbIssue, catalogue.EnrolmentVerbRedeem, catalogue.EnrolmentVerbConfirm, catalogue.EnrolmentVerbRevoke, + catalogue.EnrolmentVerbTokens, catalogue.EnrolmentVerbAgain} const ( - EventIssued = "enrolment-issued" - EventEnrolled = "node-enrolled" - EventRefused = "enrolment-refused" - EventRevoked = "enrolment-revoked" - EventExpired = "enrolment-expired" + EventIssued = catalogue.EnrolmentIssuedEvent + EventEnrolled = catalogue.NodeEnrolledEvent + EventRefused = catalogue.EnrolmentRefusedEvent + EventRevoked = catalogue.EnrolmentRevokedEvent + EventExpired = catalogue.EnrolmentExpiredEvent ) // EndEvents are the events that end an enrolment; the holder's durable consumer hears each (ADR 0587 §5) and asks the vault // to drop the token. A seat's event is heard as .. var EndEvents = []string{EventEnrolled, EventRefused, EventRevoked, EventExpired} -// Stage is how far an open enrolment has come (ADR 0587 §2). -type Stage string - +// The stages of an open enrolment (ADR 0587 §2), as the seat's contract names them (delivery.Enrolment). const ( // StageAsked: the operator's Approve is asked, no token exists. - StageAsked Stage = "asked" + StageAsked = delivery.EnrolmentAsked // StageLive: the Approve is given, the token exists and expires on the bus with the key. - StageLive Stage = "live" + StageLive = delivery.EnrolmentLive // StageRedeeming: the token is spent; the key has no expiry and ends only by confirm or revoke. - StageRedeeming Stage = "redeeming" + StageRedeeming = delivery.EnrolmentRedeeming +) + +// Enrolment and Reservation are the seat's state contract (delivery.Enrolment, delivery.EnrolmentAddress). +type ( + Enrolment = delivery.Enrolment + Reservation = delivery.EnrolmentAddress ) const ( - // DefaultExpiry and MaxExpiry bound a token's life (ADR 0587 §4). - DefaultExpiry = time.Hour - MaxExpiry = 24 * time.Hour - // EnrolUser is the bus user a joining machine dials as: enrol. (ADR 0587 §4). - enrolPrefix = "enrol." - nodePrefix = "node." + nodePrefix = "node." ) -// Enrolment is the value of the key node. (ADR 0587 §2): no secret, nothing derived from the token. -type Enrolment struct { - V int `json:"v"` - Node string `json:"node"` - Address string `json:"address"` - Why string `json:"why"` - Caller string `json:"caller"` - Ask string `json:"ask,omitempty"` - Stage Stage `json:"stage"` - Issued time.Time `json:"issued"` - Expires time.Time `json:"expires"` - // KeysDigest is the digest of the public keys the first redeem presented (ADR 0587 §5), empty before it. - KeysDigest string `json:"keys-digest,omitempty"` -} - -// Reservation is the value of the key address.
: the node's name alone (ADR 0587 §2). -type Reservation struct { - V int `json:"v"` - Node string `json:"node"` -} - // NodeKey and AddressKey are the two keys of an enrolment. -func NodeKey(node string) string { return "node." + node } -func AddressKey(address string) string { return "address." + address } +func NodeKey(node string) string { return delivery.EnrolmentNodeKey(node) } +func AddressKey(address string) string { return delivery.EnrolmentAddressKey(address) } // EnrolUser and NodeUser are the two bus users of a node: the one it joins with, and its own. -func EnrolUser(node string) string { return enrolPrefix + node } +func EnrolUser(node string) string { return delivery.EnrolmentUser(node) } func NodeUser(node string) string { return nodePrefix + node } var nodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) @@ -96,20 +78,17 @@ func CheckNode(node string) error { return nil } -func decodeEnrolment(raw []byte) (Enrolment, error) { - var e Enrolment - if err := json.Unmarshal(raw, &e); err != nil { - return Enrolment{}, fmt.Errorf("an enrolment key does not read: %w", err) +func decodeEnrolment(raw []byte) (Enrolment, error) { return delivery.DecodeEnrolment(raw) } + +// Encode is a value of the bucket as the seat's contract writes it: an Enrolment or a Reservation checked and versioned. +func Encode(v any) (json.RawMessage, error) { + switch x := v.(type) { + case Enrolment: + return delivery.EncodeEnrolment(x) + case Reservation: + return delivery.EncodeEnrolmentAddress(x.Node) } - if e.Node == "" || e.Address == "" { - return Enrolment{}, errors.New("an enrolment key names no node or no address") - } - switch e.Stage { - case StageAsked, StageLive, StageRedeeming: - default: - return Enrolment{}, fmt.Errorf("the enrolment of %s is in no stage %q", e.Node, e.Stage) - } - return e, nil + return json.Marshal(v) } // Row is a key as the store holds it. diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go index b2490d0..7da3442 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -18,20 +18,22 @@ import ( // Vault is the vault's side of enrolment (ADR 0587 §3, §5): the token and the node's account, and the signatures. The // holder keeps nothing of what it answers. type Vault interface { - // Token asks the vault to make the token and its hash for node, expiring at expires, and answers the public fields - // the joining machine must trust. No token is in the answer. - Token(node string, expires time.Time) (json.RawMessage, error) + // Token asks the vault to make the token and its hash for node, expiring at expires, and answers what the joining + // machine must trust (the bus's address and fingerprint, the vault's signing key). No token is in the answer. + Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) // Sealed asks the vault for node's token sealed to the operator key, as the vault sealed it. - Sealed(node string) (json.RawMessage, error) + Sealed(node string) (string, error) // Drop asks the vault to drop node's token and its hash; it answers only once both are gone. Drop(node string) error // Account asks for node's account sealed to the sealing key; the same node and keys again answer the same account. - Account(node, sealingKey, keysDigest string) (json.RawMessage, error) + Account(node, sealingKey, keysDigest string) (delivery.NodeAccountAnswer, error) // Retire asks the vault to retire the account it minted for node. Retire(node string) error // SignCertificate and SignReply sign through the vault's `sign`. - SignCertificate(node, servingKey string) (json.RawMessage, error) - SignReply(node string, reply []byte) (json.RawMessage, error) + SignCertificate(node, servingKey string) (delivery.SignAnswer, error) + SignReply(node string, reply []byte) (delivery.SignAnswer, error) + // Signer is the vault's public signing key in base64, which the redeem answer names for the node to check it against its token. + Signer() (string, error) } // Asker is the operator's channel as the holder asks it (asker.Client is one). @@ -99,12 +101,14 @@ type AddNode struct { Warrant Warrant `json:"warrant"` } -// NodeKeys are the public halves of a joining node's keys: identity, private network, sealing and serving. +// NodeKeys are the public halves of a joining node's keys: identity, private network, sealing and serving, and the +// tunnel it found, as delivery.Redeem presents them. type NodeKeys struct { - Public []byte `json:"public_key"` - Overlay string `json:"overlay_key,omitempty"` - Sealing string `json:"sealing_key,omitempty"` - Serving string `json:"serving_key,omitempty"` + Public string `json:"public_key"` + Overlay string `json:"overlay_key,omitempty"` + Sealing string `json:"sealing_key,omitempty"` + Serving string `json:"serving_key,omitempty"` + Tunnel json.RawMessage `json:"tunnel,omitempty"` } // Warrant is the issue's warrant as the proposal carries it: the ask the operator answered, the node and the address it @@ -115,13 +119,6 @@ type Warrant struct { Address string `json:"address"` } -// Digest is the digest of the keys a node presented, the one hash an event carries (ADR 0587 §1). -func (k NodeKeys) Digest() string { return delivery.DigestOf(k.bytes()) } - -func (k NodeKeys) bytes() []byte { - return []byte("novox-mesh-enrol-keys\x00" + string(k.Public) + "\x00" + k.Overlay + "\x00" + k.Sealing + "\x00" + k.Serving) -} - // issuing says whether issue is on: only while the controller's lease is handed to a holder that is no controller (ADR 0587 // §4), and why either way. func (h *Holder) issuing() (bool, string) { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index e701e00..86b414b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -11,6 +11,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Event is what each of the seat's events carries (ADR 0587 §2): the key's whole content, which holds no secret and nothing @@ -59,10 +60,10 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw findings = append(findings, "no why is given: the operator is asked why this machine joins") } if lasts == 0 { - lasts = DefaultExpiry + lasts = delivery.DefaultTokenLife } - if lasts < 0 || lasts > MaxExpiry { - findings = append(findings, fmt.Sprintf("the expiry %s is outside zero to %s", lasts, MaxExpiry)) + if lasts < 0 || lasts > delivery.MostTokenLife { + findings = append(findings, fmt.Sprintf("the expiry %s is outside zero to %s", lasts, delivery.MostTokenLife)) } if on, reason := h.issuing(); !on { findings = append(findings, "issuing is off until the controller's lease is written to a holder that is no controller: "+reason+ @@ -89,7 +90,7 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %s", node, strings.Join(findings, "; ")) } now := h.now() - e := Enrolment{V: 1, Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, Issued: now, Expires: now.Add(lasts)} + e := Enrolment{Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, Issued: now, Expires: now.Add(lasts)} e.Ask = "enrol-" + node + "-" + now.UTC().Format("20060102t150405") skip := map[string]bool{} for { @@ -101,7 +102,7 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw if err != nil { return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %w", node, err) } - switch err := h.Store.Create(AddressKey(addr), Reservation{V: 1, Node: node}, 0); { + switch err := h.Store.Create(AddressKey(addr), Reservation{Node: node}, 0); { case err == nil: e.Address = addr case errors.Is(err, ErrExists): @@ -242,7 +243,7 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { if _, err := h.Store.Put(NodeKey(e.Node), e, ttl); err != nil { return fmt.Errorf("the enrolment of %s cannot be moved to live: %w", e.Node, err) } - if _, err := h.Store.Put(AddressKey(e.Address), Reservation{V: 1, Node: e.Node}, ttl); err != nil { + if _, err := h.Store.Put(AddressKey(e.Address), Reservation{Node: e.Node}, ttl); err != nil { return fmt.Errorf("the address key %s cannot be given its expiry: %w", e.Address, err) } return h.sayEvent(EventIssued, e, "") diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index d42413b..4325087 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -12,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // memStore is the holder's bucket in memory: revisions, and the expiry each key was written with. @@ -29,7 +30,10 @@ func newMem() *memStore { } func (m *memStore) write(key string, v any, ttl time.Duration) uint64 { - raw, _ := json.Marshal(v) + raw, err := Encode(v) + if err != nil { + panic(err) // a test wrote a value the contract refuses + } m.rev++ m.rows[key] = Row{Value: raw, Revision: m.rev} m.ttl[key] = ttl @@ -123,23 +127,24 @@ func (v *fakeVault) do(what string) error { return nil } -func (v *fakeVault) Token(node string, expires time.Time) (json.RawMessage, error) { - return json.RawMessage(`{"broker":"b","fingerprint":"f","signer":"s"}`), v.do("token " + node) +func (v *fakeVault) Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) { + return delivery.EnrolmentTokenAnswer{Node: node, Broker: "b", Fingerprint: "f", Signer: "s"}, v.do("token " + node) } -func (v *fakeVault) Sealed(node string) (json.RawMessage, error) { - return json.RawMessage(`"sealed-for-` + node + `"`), v.do("sealed " + node) +func (v *fakeVault) Sealed(node string) (string, error) { + return "sealed-for-" + node, v.do("sealed " + node) } func (v *fakeVault) Drop(node string) error { return v.do("drop " + node) } -func (v *fakeVault) Account(node, sealing, digest string) (json.RawMessage, error) { - return json.RawMessage(`{"account":"` + node + `"}`), v.do("account " + node) +func (v *fakeVault) Account(node, sealing, digest string) (delivery.NodeAccountAnswer, error) { + return delivery.NodeAccountAnswer{Node: node, Login: "node." + node, Account: "account-" + node}, v.do("account " + node) } func (v *fakeVault) Retire(node string) error { return v.do("retire " + node) } -func (v *fakeVault) SignCertificate(node, serving string) (json.RawMessage, error) { - return json.RawMessage(`{"certificate":"c"}`), v.do("certificate " + node) +func (v *fakeVault) SignCertificate(node, serving string) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Certificate: "PEM-" + serving}, v.do("certificate " + node) } -func (v *fakeVault) SignReply(node string, reply []byte) (json.RawMessage, error) { - return json.RawMessage(`{"signature":"sig","signer":"fp"}`), v.do("sign " + node) +func (v *fakeVault) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindEnrolment, Node: node, Signature: "sig-" + delivery.DigestOf(reply)[7:15], Signer: "fp"}, v.do("sign " + node) } +func (v *fakeVault) Signer() (string, error) { return "vault-signer", nil } type fakeAsker struct { mu sync.Mutex diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go index 3ff7ccd..99cb6b0 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -30,11 +30,14 @@ func (runtimeStore) Get(key string) (Row, bool, error) { } func (runtimeStore) Create(key string, v any, ttl time.Duration) error { - var err error + raw, err := Encode(v) + if err != nil { + return err + } if ttl > 0 { - _, err = stdio.State(State).CreateTTL(key, v, ttl) + _, err = stdio.State(State).CreateTTL(key, raw, ttl) } else { - _, err = stdio.State(State).Create(key, v) + _, err = stdio.State(State).Create(key, raw) } if errors.Is(err, stdio.ErrChanged) { return ErrExists @@ -43,7 +46,11 @@ func (runtimeStore) Create(key string, v any, ttl time.Duration) error { } func (runtimeStore) Update(key string, v any, revision uint64) (uint64, error) { - rev, err := stdio.State(State).Update(key, v, revision) + raw, err := Encode(v) + if err != nil { + return 0, err + } + rev, err := stdio.State(State).Update(key, raw, revision) if errors.Is(err, stdio.ErrChanged) { return 0, ErrChanged } @@ -51,10 +58,14 @@ func (runtimeStore) Update(key string, v any, revision uint64) (uint64, error) { } func (runtimeStore) Put(key string, v any, ttl time.Duration) (uint64, error) { - if ttl > 0 { - return stdio.State(State).PutTTL(key, v, ttl) + raw, err := Encode(v) + if err != nil { + return 0, err } - return stdio.State(State).Put(key, v) + if ttl > 0 { + return stdio.State(State).PutTTL(key, raw, ttl) + } + return stdio.State(State).Put(key, raw) } func (runtimeStore) Delete(key string) error { return stdio.State(State).Delete(key) } @@ -82,7 +93,7 @@ func newHolder() *Holder { rec := handacts.Recorder{Ask: stdio.Ask} rd := runtimeReader{} return &Holder{ - Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call}, Asker: asker.New("asked"), + Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call, read: rd}, Asker: asker.New("asked"), Say: func(event string, body any, id string) error { return stdio.SeatPublish(SeatEventSubject(event), body, id) }, diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go index d714e4f..e2a01a0 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -3,51 +3,16 @@ package main import ( "crypto/ed25519" "encoding/base64" - "encoding/json" "errors" "fmt" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) -// RedeemArgs are what a joining node-engine says at redeem (ADR 0587 §6): the public halves of the keys it made where it -// runs, and a proof, signed with its identity key, that it holds the private half of the first. -type RedeemArgs struct { - Node string `json:"node"` - PublicKey []byte `json:"public_key"` - OverlayKey string `json:"overlay_key,omitempty"` - SealingKey string `json:"sealing_key,omitempty"` - ServingKey string `json:"serving_key,omitempty"` - Profile map[string]any `json:"profile,omitempty"` - Proof []byte `json:"proof"` -} - -// Keys are the public halves presented. -func (a RedeemArgs) Keys() NodeKeys { - return NodeKeys{Public: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey} -} - -// RedeemProof is the bytes a node signs with its identity key when it redeems: the node's name and every key it presents, so a -// proof cannot be moved to another request or another node. The layout is mesh-host's EnrolProof (internal/link/enrol.go), -// with the node's name in the place of the token's secret, which this seat never holds. -func RedeemProof(node string, k NodeKeys) []byte { - return []byte("novox-mesh-enrol\x00" + node + "\x00" + base64.StdEncoding.EncodeToString(k.Public) + - "\x00" + k.Overlay + "\x00" + k.Sealing + "\x00" + k.Serving) -} - -// RedeemReply is what the vault signs: the node's account sealed to its sealing key, the signed certificate for its internal -// name and the address the operator approved. -type RedeemReply struct { - Node string `json:"node"` - Address string `json:"address"` - Account json.RawMessage `json:"account"` - Certificate json.RawMessage `json:"certificate"` -} - -// RedeemAnswer is what redeem answers: the reply's bytes and the vault's signature over them. The node-engine refuses an -// answer not signed by the key its token carries. -type RedeemAnswer struct { - Reply []byte `json:"reply"` - Signed json.RawMessage `json:"signed"` +// keysOf are the public halves a redeem presented. +func keysOf(a delivery.Redeem) NodeKeys { + return NodeKeys{Public: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey, Tunnel: a.Tunnel} } // Redeem is the joining node-engine's one call (ADR 0587 §5, §6). It answers only a caller the runtime proves to be @@ -56,51 +21,61 @@ type RedeemAnswer struct { // compare-and-set, with no expiry), asks the vault for the node's account and the certificate, writes the node into placement, // waits for the bus node to load node., and answers the whole, signed by the vault. Called again with the same keys it // finishes what is not done and answers the same. -func (h *Holder) Redeem(caller string, a RedeemArgs) (RedeemAnswer, error) { +func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer, error) { if err := CheckNode(a.Node); err != nil { - return RedeemAnswer{}, err + return delivery.RedeemAnswer{}, err } if caller != EnrolUser(a.Node) { - return RedeemAnswer{}, fmt.Errorf("only %s may redeem the enrolment of %s, and the runtime proved the caller to be %q", EnrolUser(a.Node), a.Node, caller) + return delivery.RedeemAnswer{}, fmt.Errorf("only %s may redeem the enrolment of %s, and the runtime proved the caller to be %q", EnrolUser(a.Node), a.Node, caller) } - keys := a.Keys() - if len(a.PublicKey) != ed25519.PublicKeySize || !ed25519.Verify(a.PublicKey, RedeemProof(a.Node, keys), a.Proof) { - return RedeemAnswer{}, errors.New("the proof does not verify against the identity key presented: the enrolment is not spent") + pub, err := base64.StdEncoding.DecodeString(a.PublicKey) + proof, perr := a.ProofBytes() + if err != nil || perr != nil || len(pub) != ed25519.PublicKeySize || + !ed25519.Verify(pub, delivery.RedeemProof(a.Node, a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey), proof) { + return delivery.RedeemAnswer{}, errors.New("the proof does not verify against the identity key presented: the enrolment is not spent") } - digest := keys.Digest() + if a.SealingKey == "" { + return delivery.RedeemAnswer{}, errors.New("no sealing key is presented, so the account cannot be sealed to the node: the enrolment is not spent") + } + digest := a.KeysDigest() e, err := h.spend(a.Node, digest) if err != nil { - return RedeemAnswer{}, err + return delivery.RedeemAnswer{}, err } account, err := h.Vault.Account(e.Node, a.SealingKey, digest) if err != nil { - return RedeemAnswer{}, fmt.Errorf("the vault did not give the account of %s (call redeem again with the same keys): %w", e.Node, err) + return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not give the account of %s (call redeem again with the same keys): %w", e.Node, err) } cert, err := h.Vault.SignCertificate(e.Node, a.ServingKey) if err != nil { - return RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) + return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) } - if err := h.place(e, a, keys); err != nil { - return RedeemAnswer{}, err + if err := h.place(e, keysOf(a)); err != nil { + return delivery.RedeemAnswer{}, err } if h.Loaded != nil { loaded, err := h.Loaded(NodeUser(e.Node)) if err != nil { - return RedeemAnswer{}, fmt.Errorf("whether the bus node loaded %s cannot be read (call redeem again with the same keys): %w", NodeUser(e.Node), err) + return delivery.RedeemAnswer{}, fmt.Errorf("whether the bus node loaded %s cannot be read (call redeem again with the same keys): %w", NodeUser(e.Node), err) } if !loaded { - return RedeemAnswer{}, fmt.Errorf("the bus node has not loaded %s yet (call redeem again with the same keys)", NodeUser(e.Node)) + return delivery.RedeemAnswer{}, fmt.Errorf("the bus node has not loaded %s yet (call redeem again with the same keys)", NodeUser(e.Node)) } } - reply, err := json.Marshal(RedeemReply{Node: e.Node, Address: e.Address, Account: account, Certificate: cert}) + signer, err := h.Vault.Signer() if err != nil { - return RedeemAnswer{}, err + return delivery.RedeemAnswer{}, fmt.Errorf("the vault's signing key cannot be named (call redeem again with the same keys): %w", err) } - signed, err := h.Vault.SignReply(e.Node, reply) + out := delivery.RedeemAnswer{Node: e.Node, Login: account.Login, Account: account.Account, Certificate: cert.Certificate, Signer: signer} + if out.Login == "" { + out.Login = NodeUser(e.Node) + } + signed, err := h.Vault.SignReply(e.Node, out.SignedBytes()) if err != nil { - return RedeemAnswer{}, fmt.Errorf("the vault did not sign the answer (call redeem again with the same keys): %w", err) + return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the answer (call redeem again with the same keys): %w", err) } - return RedeemAnswer{Reply: reply, Signed: signed}, nil + out.Signature = signed.Signature + return out, nil } // spend moves the key from live to redeeming with the presented keys' digest, by compare-and-set, before its expiry; at @@ -138,7 +113,7 @@ func (h *Holder) spend(node, digest string) (Enrolment, error) { return Enrolment{}, fmt.Errorf("the enrolment of %s cannot be spent: %w", node, err) } // The move to redeeming rewrites both keys without an expiry, so a redeem under way never loses its key (§5). - if _, err := h.Store.Put(AddressKey(e.Address), Reservation{V: 1, Node: node}, 0); err != nil { + if _, err := h.Store.Put(AddressKey(e.Address), Reservation{Node: node}, 0); err != nil { return Enrolment{}, fmt.Errorf("the address key %s cannot be kept (call redeem again with the same keys): %w", e.Address, err) } return e, nil @@ -147,7 +122,7 @@ func (h *Holder) spend(node, digest string) (Enrolment, error) { } // place writes the node into placement with the issue's warrant, unless placement holds it at the approved address already. -func (h *Holder) place(e Enrolment, a RedeemArgs, keys NodeKeys) error { +func (h *Holder) place(e Enrolment, keys NodeKeys) error { placements, err := h.placements() if err != nil { return fmt.Errorf("placement cannot be read (call redeem again with the same keys): %w", err) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go index 0d3e2e5..5d173d9 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -6,20 +6,16 @@ import ( "encoding/base64" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) -// engineEnrolProof is mesh-host's EnrolProof (internal/link/enrol.go:112-114), copied: the bytes a node-engine signs. The -// holder's RedeemProof must be the same bytes with the node's name in the token's place. -func engineEnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { - return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + - "\x00" + overlay + "\x00" + sealing + "\x00" + serving) -} - // redeemArgs are a node's redeem call with a proof signed by the key seeded, over the keys given. -func redeemArgs(node string, seed byte, serving string) RedeemArgs { +func redeemArgs(node string, seed byte, serving string) delivery.Redeem { priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{seed}, ed25519.SeedSize)) - a := RedeemArgs{Node: node, PublicKey: priv.Public().(ed25519.PublicKey), OverlayKey: "overlay-" + node, SealingKey: "sealing-" + node, ServingKey: serving} - a.Proof = ed25519.Sign(priv, RedeemProof(node, a.Keys())) + a := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(priv.Public().(ed25519.PublicKey)), + OverlayKey: "overlay-" + node, SealingKey: "sealing-" + node, ServingKey: serving} + a.Proof = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, delivery.RedeemProof(node, a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey))) return a } @@ -29,15 +25,13 @@ func redeemArgs(node string, seed byte, serving string) RedeemArgs { func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { good := redeemArgs("ace", 1, "serving-a") bad := good - bad.Proof = append([]byte{}, good.Proof...) - bad.Proof[0] ^= 1 - if !bytes.Equal(RedeemProof("ace", good.Keys()), engineEnrolProof("ace", good.PublicKey, good.OverlayKey, good.SealingKey, good.ServingKey)) { - t.Fatal("the proof's bytes are not the engine's") - } + raw, _ := good.ProofBytes() + raw[0] ^= 1 + bad.Proof = base64.StdEncoding.EncodeToString(raw) cases := []struct { name string caller string - args RedeemArgs + args delivery.Redeem setup func(r *rig) want string }{ @@ -46,7 +40,7 @@ func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { {"after the expiry", "enrol.ace", good, func(r *rig) { r.now = r.now.Add(2 * time.Hour) }, "expired at"}, {"other keys than the first", "enrol.ace", redeemArgs("ace", 2, "serving-b"), func(r *rig) { e, _ := r.enrolment("ace") - e.Stage, e.KeysDigest = StageRedeeming, good.Keys().Digest() + e.Stage, e.KeysDigest = StageRedeeming, good.KeysDigest() _, _ = r.st.Put(NodeKey("ace"), e, 0) }, "spent once"}, {"before the operator's yes", "enrol.ace", good, func(r *rig) { @@ -80,10 +74,10 @@ func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { t.Fatal(err) } e, _ := r.enrolment("ace") - if e.Stage != StageRedeeming || e.KeysDigest != good.Keys().Digest() || r.st.ttl[NodeKey("ace")] != 0 || r.st.ttl[AddressKey("10.9.0.2")] != 0 { + if e.Stage != StageRedeeming || e.KeysDigest != good.KeysDigest() || r.st.ttl[NodeKey("ace")] != 0 || r.st.ttl[AddressKey("10.9.0.2")] != 0 { t.Errorf("spent %+v, ttl %v/%v", e, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")]) } - if len(r.added) != 1 || r.added[0].Warrant != (Warrant{Ask: "ask-ace", Node: "ace", Address: "10.9.0.2"}) || len(ans.Reply) == 0 || len(ans.Signed) == 0 { + if len(r.added) != 1 || r.added[0].Warrant != (Warrant{Ask: "ask-ace", Node: "ace", Address: "10.9.0.2"}) || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { t.Errorf("placement %+v, answer %+v", r.added, ans) } r.loaded = false @@ -110,7 +104,7 @@ func TestARedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { t.Fatalf("%s: no error", stop) } got, err := r.h.Redeem("enrol.ace", good) - if err != nil || !bytes.Equal(got.Reply, want.Reply) || string(got.Signed) != string(want.Signed) { + if err != nil || got != want { t.Errorf("%s: %v, answer %+v, want %+v", stop, err, got, want) } if stop == "sign ace" && len(r.added) != 1 { @@ -121,7 +115,7 @@ func TestARedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { r := newRig(t) r.seed("ace", "10.9.0.2", StageRedeeming) e, _ := r.enrolment("ace") - e.KeysDigest = good.Keys().Digest() + e.KeysDigest = good.KeysDigest() _, _ = r.st.Put(NodeKey("ace"), e, 0) r.rd["mesh-delivery-assigner.placement"]["ace.node"] = `{"v":1,"node":"ace","decided":{"address":"10.9.0.2"},"modules":[]}` if _, err := r.h.Redeem("enrol.ace", good); err != nil || len(r.added) != 0 { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go index df23f1e..a153797 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go @@ -10,18 +10,18 @@ import ( // TokenRow is one open enrolment as `tokens` answers it (ADR 0587 §8): its stage, expiry, ask and revision, and for a node // at stage live the token sealed to the operator key as the vault sealed it (the holder never reads it and keeps it nowhere). type TokenRow struct { - Node string `json:"node"` - Address string `json:"address"` - Why string `json:"why"` - Caller string `json:"caller"` - Stage Stage `json:"stage"` - Ask string `json:"ask,omitempty"` - Issued time.Time `json:"issued"` - Expires time.Time `json:"expires"` - Age string `json:"age"` - Revision uint64 `json:"revision"` - Keys string `json:"keys-digest,omitempty"` - Sealed json.RawMessage `json:"sealed,omitempty"` + Node string `json:"node"` + Address string `json:"address"` + Why string `json:"why"` + Caller string `json:"caller"` + Stage string `json:"stage"` + Ask string `json:"ask,omitempty"` + Issued time.Time `json:"issued"` + Expires time.Time `json:"expires"` + Age string `json:"age"` + Revision uint64 `json:"revision"` + Keys string `json:"keys-digest,omitempty"` + Sealed string `json:"sealed,omitempty"` // Unsealed says why no sealed token is given for a node that should have one. Unsealed string `json:"unsealed,omitempty"` } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index 0ca284c..ab4b890 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -7,17 +7,19 @@ import ( "slices" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // seed writes an open enrolment of node at stage, with its address key, as the holder would have. -func (r *rig) seed(node, address string, stage Stage) { - e := Enrolment{V: 1, Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, Stage: stage, +func (r *rig) seed(node, address string, stage string) { + e := Enrolment{Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, Stage: stage, Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour)} if stage == StageRedeeming { - e.KeysDigest = "sha256:keys" + e.KeysDigest = delivery.KeysDigest("pk", "ov", "se", "sv") } _, _ = r.st.Put(NodeKey(node), e, 0) - _, _ = r.st.Put(AddressKey(address), Reservation{V: 1, Node: node}, 0) + _, _ = r.st.Put(AddressKey(address), Reservation{Node: node}, 0) } // 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, the fill and whether issuing is on; @@ -34,7 +36,7 @@ func TestTokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { var stages []string for _, row := range all.Open { stages = append(stages, row.Node+":"+string(row.Stage)) - if row.Revision == 0 || row.Expires.IsZero() || row.Ask == "" || row.Sealed != nil { + if row.Revision == 0 || row.Expires.IsZero() || row.Ask == "" || row.Sealed != "" { t.Errorf("row %+v lacks revision, expiry or ask, or relays a token for a list of all", row) } } @@ -42,7 +44,7 @@ func TestTokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { t.Errorf("tokens %v, fill %+v, issuing %+v", stages, all.Fill, all.Issuing) } one, err := r.h.Tokens("g14", false) - if err != nil || len(one.Open) != 1 || string(one.Open[0].Sealed) != `"sealed-for-g14"` { + if err != nil || len(one.Open) != 1 || one.Open[0].Sealed != "sealed-for-g14" { t.Fatalf("one node: %+v, %v", one, err) } for k, row := range r.st.rows { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go index 1c1ca47..23a73dc 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -7,6 +7,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Tool names: the seat's verbs, as the manifest claims them (ADR 0587 §1, §8). @@ -38,17 +39,8 @@ func flag(a map[string]any, k string) bool { return a[k] == true || strings.EqualFold(arg(a, k), "true") } -// lastsOf reads an expiry as a duration; none is the default. -func lastsOf(s string) (time.Duration, error) { - if s == "" { - return 0, nil - } - d, err := time.ParseDuration(s) - if err != nil { - return 0, fmt.Errorf("the expiry %q is no duration (for example 1h): %w", s, err) - } - return d, nil -} +// lastsOf reads an expiry as a duration (delivery.TokenLife); none is the default. +func lastsOf(s string) (time.Duration, error) { return delivery.TokenLife(s) } // Tools are what the runtime serves. func Tools(h *Holder) []stdio.Tool { @@ -79,7 +71,7 @@ func Tools(h *Holder) []stdio.Tool { "overlay_key": str("its private-network key"), "sealing_key": str("its sealing key, base64"), "serving_key": str("its serving key"), "proof": str("the proof, base64")}, "node", "public_key", "proof"), Run: func(a map[string]any) (any, error) { - var args RedeemArgs + var args delivery.Redeem raw, err := json.Marshal(a) if err != nil { return nil, err diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go index b96269f..29c1f65 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go @@ -3,6 +3,7 @@ package main import ( "encoding/base64" "encoding/json" + "errors" "fmt" "time" @@ -12,47 +13,83 @@ import ( // Seats the holder calls (ADR 0587 §3, §5, §7). The vault's two verbs are the ones its manifest names this seat the only // caller of; its `sign` names this seat for a node's certificate and a redeem answer. const ( - vaultToken = "seat:mesh-vault.enrolment-token" - vaultAccount = "seat:mesh-vault.node-account" - vaultSign = "seat:mesh-vault.sign" + vaultToken = "seat:mesh-vault." + delivery.EnrolmentTokenVerb + vaultAccount = "seat:mesh-vault." + delivery.NodeAccountVerb + vaultSign = "seat:mesh-vault." + delivery.SignVerb placementAdd = "seat:mesh-placement.propose" ) // Call asks a seat verb by its address (stdio.Ask through the runtime, or a test's). type Call func(key string, body any) (json.RawMessage, error) -// vaultOver is the vault as the holder reaches it: the seat's verbs over a Call. -type vaultOver struct{ call Call } - -func (v vaultOver) Token(node string, expires time.Time) (json.RawMessage, error) { - return v.call(vaultToken, map[string]any{"node": node, "expires": expires.UTC().Format(time.RFC3339)}) +// vaultOver is the vault as the holder reaches it: the seat's verbs over a Call, and its public signing key from its state. +type vaultOver struct { + call Call + read Reader } -func (v vaultOver) Sealed(node string) (json.RawMessage, error) { - return v.call(vaultToken, map[string]any{"node": node, "sealed": true}) +func answer[T any](raw json.RawMessage, err error) (T, error) { + var out T + if err != nil { + return out, err + } + if uerr := json.Unmarshal(raw, &out); uerr != nil { + return out, fmt.Errorf("the vault's answer does not read: %w", uerr) + } + return out, nil +} + +func (v vaultOver) Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) { + return answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "expires": expires.UTC().Format(time.RFC3339)})) +} + +func (v vaultOver) Sealed(node string) (string, error) { + a, err := answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "sealed": "true"})) + if err == nil && a.Sealed == "" { + err = errors.New("the vault gave no sealed token") + } + return a.Sealed, err } func (v vaultOver) Drop(node string) error { - _, err := v.call(vaultToken, map[string]any{"node": node, "drop": true}) + a, err := answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "drop": "true"})) + if err == nil && !a.Dropped { + err = errors.New("the vault did not say the token and its hash are gone") + } return err } -func (v vaultOver) Account(node, sealingKey, keysDigest string) (json.RawMessage, error) { - return v.call(vaultAccount, map[string]any{"node": node, "sealing_key": sealingKey, "keys_digest": keysDigest}) +func (v vaultOver) Account(node, sealingKey, keysDigest string) (delivery.NodeAccountAnswer, error) { + return answer[delivery.NodeAccountAnswer](v.call(vaultAccount, map[string]any{"node": node, "sealing_key": sealingKey, "keys_digest": keysDigest})) } func (v vaultOver) Retire(node string) error { - _, err := v.call(vaultAccount, map[string]any{"node": node, "retire": true}) + a, err := answer[delivery.NodeAccountAnswer](v.call(vaultAccount, map[string]any{"node": node, "retire": "true"})) + if err == nil && !a.Retired { + err = errors.New("the vault did not say the account is retired") + } return err } -func (v vaultOver) SignCertificate(node, servingKey string) (json.RawMessage, error) { - return v.call(vaultSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node, "serving_key": servingKey}) +func (v vaultOver) SignCertificate(node, servingKey string) (delivery.SignAnswer, error) { + return answer[delivery.SignAnswer](v.call(vaultSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node, "serving_key": servingKey})) } -func (v vaultOver) SignReply(node string, reply []byte) (json.RawMessage, error) { - return v.call(vaultSign, map[string]any{"kind": delivery.KindEnrolment, "node": node, - "reply": base64.StdEncoding.EncodeToString(reply)}) +func (v vaultOver) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { + return answer[delivery.SignAnswer](v.call(vaultSign, map[string]any{"kind": delivery.KindEnrolment, "node": node, + "reply": base64.StdEncoding.EncodeToString(reply)})) +} + +func (v vaultOver) Signer() (string, error) { + raw, found, err := v.read.Get(delivery.SignersRead, delivery.SignersKeyKey) + if err != nil || !found { + return "", fmt.Errorf("the vault's signing key is not readable in %s: %v", delivery.SignersRead, err) + } + k, err := delivery.DecodeSignerKey(raw) + if err != nil { + return "", err + } + return k.Key, nil } // addNodeOver asks mesh-placement to add the node with the issue's warrant (ADR 0587 §7). diff --git a/modules/mesh-enrolment/module.json b/modules/mesh-enrolment/module.json index 4504a55..29c1455 100644 --- a/modules/mesh-enrolment/module.json +++ b/modules/mesh-enrolment/module.json @@ -37,6 +37,7 @@ "mesh-controller.lease", "mesh-delivery-assigner.placement", "mesh-grants.user-list", + "mesh-vault.signers", "node-engine.have" ], "state": [ From 447bcf3a6b885b01f58a29751418f354e745e1bf Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:27:54 +0100 Subject: [PATCH 07/20] The assigner takes an add-node from mesh-enrolment and writes the joining node with no ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit propose takes the enrolment's warrant {node, address, ask, warrant} and refuses an add-node from any caller but mesh-enrolment, with no warrant or with a warrant naming another node or address (delivery.AddNodeRefusal); the joining node counts as having a record for this one change, as ADR 0587 §7 changes ADR 0567 §2. The assigner's process then writes placement. at once, with no node to answer and no ask raised, and says placement-changed carrying the enrolment's warrant; the hand-act record of that warrant is the enrolment's, made when it was given. The holder of mesh-enrolment calls propose with the change, the keys and the warrant read from the router's record of the ask. Tests: rows appended to the seat's verbs test and a case to the whole-seat integration test; none added, so none to pay for (the budget is paid in 774/S60). (issue 774, 774/S59) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../integration_test.go | 26 +++++++++++ .../cmd/mesh-delivery-assigner/seat.go | 38 ++++++++++++++++ .../cmd/mesh-delivery-assigner/seat_tools.go | 43 ++++++++++++++++++- .../mesh-delivery-assigner/seat_tools_test.go | 21 +++++++++ .../internal/assign/seat.go | 16 ++++++- .../cmd/mesh-enrolment/holder.go | 35 ++++----------- .../cmd/mesh-enrolment/kit_test.go | 14 +++--- .../mesh-enrolment/cmd/mesh-enrolment/main.go | 22 ++++++++-- .../cmd/mesh-enrolment/redeem.go | 21 ++++----- .../cmd/mesh-enrolment/redeem_test.go | 3 +- .../cmd/mesh-enrolment/vault.go | 7 +-- 11 files changed, 193 insertions(+), 53 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 9e48b6a..ec6fb24 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1133,4 +1133,30 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") { t.Errorf("the silent node's verdict = %+v", failed.Verdict) } + + // 774/S59 (ADR 0587 §7): once the lease is handed away, a joining node is written on the enrolment's warrant with no node to + // answer and no ask raised, and placement-changed carries that warrant. + b.putJSON(t, buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-grants","handed-by":"operator"}`)) + for len(heard) > 0 { + <-heard + } + joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`, + "changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"identity":"aWQ=","sealing":"c2Vh"}}]`}, "novox.mesh-enrolment") + if err != nil { + t.Fatal(err) + } + select { + case e := <-heard: + if e.Proposal.ID != joined.(proposed).Proposal.ID || e.Warrant == nil || e.Warrant.Warrant != "enrol-ace-1@2026-10-28T20:00:00Z" || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" { + t.Errorf("placement-changed for the joining node = %+v", e) + } + case <-time.After(15 * time.Second): + t.Fatal("no placement-changed heard for the joining node") + } + if ace, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") { + t.Errorf("the joining node's key = %v, %v", ace, err) + } + if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(delivery.PlacementProposalsBucket, joined.(proposed).Proposal.ID) { + t.Error("an ask was raised for the joining node, or its proposal stays open") + } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index c8ef722..3dc3358 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -268,6 +268,10 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone") return } + if prop.Warrant != nil { + p.addNode(ctx, prop, revs) + return + } akey, _ := delivery.PlacementAskKey(prop.ID) again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At)) if prop.State == delivery.ProposalAsked { @@ -544,6 +548,40 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, p.followWrite(prop, answer, changes) } +// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve +// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a +// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does +// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so. +func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) { + current, _ := p.seatPlacements() + if moved(prop, revs) { + p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again") + return + } + laid, findings := delivery.LayChanges(current, prop.Changes) + node := prop.Changes[0].Node + value, ok := laid[node] + if len(findings) > 0 || !ok { + p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; ")) + return + } + rev, err := p.writeNode(ctx, node, &value) + if err != nil { + p.endProposal(ctx, prop, "the node's key was not written: "+err.Error()) + return + } + c := delivery.ChangeOf(delivery.Placement{Node: node}, value) + c.Before, c.After = prop.Base[node], rev + p.deleteCandidates(ctx, prop.ID) + p.deletePulses(ctx, prop.ID) + _ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID) + p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil) + w := prop.Warrant + p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant, + Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)}, + []delivery.PlacementChange{c}) +} + // followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the // controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774). func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 15a565e..2ef4889 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -217,9 +217,16 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { if err != nil { return nil, err } + warrant, err := warrantOf(args["warrant"]) + if err != nil { + return nil, err + } now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, - Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded}) + Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant}) + if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" { + findings = append(findings, refusal) + } if len(findings) > 0 { return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; ")) } @@ -229,6 +236,40 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil } +// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none. +func warrantOf(v any) (*delivery.AddNodeWarrant, error) { + var raw []byte + switch w := v.(type) { + case nil: + return nil, nil + case string: + if strings.TrimSpace(w) == "" { + return nil, nil + } + raw = []byte(w) + default: + var err error + if raw, err = json.Marshal(w); err != nil { + return nil, err + } + } + var out delivery.AddNodeWarrant + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(&out); err != nil { + return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err) + } + return &out, nil +} + +// moduleOfCaller is the module a caller is, as the runtime proves it: a module's bus user is `.`. +func moduleOfCaller(by string) string { + if _, module, ok := strings.Cut(by, "."); ok { + return module + } + return by +} + // doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says // placement-withdrawn carrying it whole. func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 6e1d736..82142d1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -150,6 +150,27 @@ func TestTheSeatsVerbs(t *testing.T) { if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil { t.Errorf("a hold: %v", err) } + // 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with + // the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused. + addNode := func(by, warrant string) error { + _, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace", + "decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"identity": "aWQ=", "sealing": "c2Vh"}}}}, by) + return err + } + good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}` + for _, c := range []struct{ by, warrant, want string }{ + {"novox.mesh-grants", good, "mesh-enrolment's alone"}, + {"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"}, + {"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"}, + {"novox.mesh-enrolment", "", "carries no warrant"}, + } { + if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want) + } + } + if err := addNode("novox.mesh-enrolment", good); err != nil { + t.Errorf("an add-node from mesh-enrolment: %v", err) + } if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") { t.Errorf("a hand-back: %v", err) } diff --git a/modules/mesh-delivery-assigner/internal/assign/seat.go b/modules/mesh-delivery-assigner/internal/assign/seat.go index 435ac4e..3b2dc1e 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat.go @@ -24,6 +24,8 @@ type ProposeIn struct { Revisions map[string]uint64 Held func(module string) bool Recorded func(node string) bool + // Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one. + Warrant *delivery.AddNodeWarrant } // Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision), @@ -33,7 +35,7 @@ type ProposeIn struct { // record). A key whose value the proposal leaves as it is changes nothing and is not laid. func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) { p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(), - Changes: in.Changes, Base: map[string]uint64{}, State: delivery.ProposalAnswering} + Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering} var findings []string if p.By == "" { findings = append(findings, "the runtime names no caller: a proposal is made by someone") @@ -75,6 +77,18 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla if recorded == nil { recorded = func(string) bool { return false } } + // A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by + // this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2). + joining := map[string]bool{} + for _, c := range in.Changes { + if c.Op == delivery.OpAddNode { + joining[c.Node] = true + } + } + if len(joining) > 0 { + was := recorded + recorded = func(node string) bool { return joining[node] || was(node) } + } for _, r := range delivery.CheckPlacements(all, held, recorded) { findings = append(findings, r) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go index 7da3442..f6dbdde 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -55,7 +55,9 @@ type Holder struct { // Warrant and Act record in the hand-act log, through the seat mesh-hand-acts: the operator's warrant for an ask, and an // act done by hand (a revoke). Warrant func(ask string) error - Act func(what, why, cause string) error + // WarrantOf is the warrant's id as the teller wrote it (the ask's id and when), read from the router's record of the ask. + WarrantOf func(ask string) (string, error) + Act func(what, why, cause string) error // Loaded says whether the bus node loaded the bus user (ADR 0587 §3: redeem answers once node. is in its list). Loaded func(user string) (bool, error) Now func() time.Time @@ -91,32 +93,13 @@ func (h *Holder) say(event string, body any, id string) error { return h.Say(event, body, id) } -// AddNode is what the holder asks mesh-placement to add (ADR 0587 §7): the node, its address, the public halves of its -// keys, and the issue's warrant, which names this node and this address. +// AddNode is what the holder asks mesh-placement to add (ADR 0587 §7): the proposal's one change, a joining node with the +// address the operator approved and the public halves of its keys, and the issue's warrant, which names this node and this +// address (delivery.AddNodeWarrant). type AddNode struct { - Node string `json:"node"` - Address string `json:"address"` - Why string `json:"why"` - Keys NodeKeys `json:"keys"` - Warrant Warrant `json:"warrant"` -} - -// NodeKeys are the public halves of a joining node's keys: identity, private network, sealing and serving, and the -// tunnel it found, as delivery.Redeem presents them. -type NodeKeys struct { - Public string `json:"public_key"` - Overlay string `json:"overlay_key,omitempty"` - Sealing string `json:"sealing_key,omitempty"` - Serving string `json:"serving_key,omitempty"` - Tunnel json.RawMessage `json:"tunnel,omitempty"` -} - -// Warrant is the issue's warrant as the proposal carries it: the ask the operator answered, the node and the address it -// named. -type Warrant struct { - Ask string `json:"ask"` - Node string `json:"node"` - Address string `json:"address"` + Why string + Change delivery.PlacementProposalChange + Warrant delivery.AddNodeWarrant } // issuing says whether issue is on: only while the controller's lease is handed to a holder that is no controller (ADR 0587 diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 4325087..3ace80b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -211,15 +211,17 @@ func newRig(t *testing.T) *rig { AddNode: func(a AddNode) error { r.added = append(r.added, a) if r.placeErr == nil { // placement writes the node as the assigner does on the warrant - r.rd["mesh-delivery-assigner.placement"][a.Node+".node"] = `{"v":1,"node":"` + a.Node + `","decided":{"address":"` + a.Address + `"},"modules":[]}` + n := a.Change.Node + r.rd["mesh-delivery-assigner.placement"][n+".node"] = `{"v":1,"node":"` + n + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` } return r.placeErr }, - Warrant: func(ask string) error { r.warr = append(r.warr, ask); return nil }, - Act: func(what, why, cause string) error { r.acts = append(r.acts, what); return nil }, - Loaded: func(string) (bool, error) { return r.loaded, nil }, - Now: func() time.Time { return r.now }, - Logf: func(f string, a ...any) { t.Logf(f, a...) }} + Warrant: func(ask string) error { r.warr = append(r.warr, ask); return nil }, + WarrantOf: func(ask string) (string, error) { return ask + "@2026-11-02T10:00:00Z", nil }, + Act: func(what, why, cause string) error { r.acts = append(r.acts, what); return nil }, + Loaded: func(string) (bool, error) { return r.loaded, nil }, + Now: func() time.Time { return r.now }, + Logf: func(f string, a ...any) { t.Logf(f, a...) }} return r } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go index 99cb6b0..a9ae270 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -92,13 +92,15 @@ func newHolder() *Holder { call := Call(stdio.Ask) rec := handacts.Recorder{Ask: stdio.Ask} rd := runtimeReader{} + client := asker.New("asked") return &Holder{ - Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call, read: rd}, Asker: asker.New("asked"), + Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call, read: rd}, Asker: client, Say: func(event string, body any, id string) error { return stdio.SeatPublish(SeatEventSubject(event), body, id) }, - AddNode: addNodeOver(call), - Warrant: func(ask string) error { _, err := rec.RecordWarrant(Module, ask); return err }, + AddNode: addNodeOver(call), + WarrantOf: warrantOf(client), + Warrant: func(ask string) error { _, err := rec.RecordWarrant(Module, ask); return err }, Act: func(what, why, cause string) error { _, err := rec.Record(handacts.Request{What: what, Why: why, Cause: cause}) return err @@ -110,6 +112,20 @@ func newHolder() *Holder { } } +// warrantOf is the warrant id the teller writes, `@`, read from the router's record of the ask. +func warrantOf(c *asker.Client) func(string) (string, error) { + return func(ask string) (string, error) { + state, w, err := c.Where(ask) + if err != nil { + return "", err + } + if w == nil { + return "", fmt.Errorf("the router records the ask %s as %q, with no warrant", ask, state) + } + return ask + "@" + w.At.UTC().Format(time.RFC3339Nano), nil + } +} + // loadedVia says a bus user is loaded when the grants part's last pass lists it among the composed users and the bus node's // loaded list holds it (its credentials row says loaded). func loadedVia(rd Reader) func(string) (bool, error) { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go index e2a01a0..38f38af 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -10,11 +10,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// keysOf are the public halves a redeem presented. -func keysOf(a delivery.Redeem) NodeKeys { - return NodeKeys{Public: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey, Tunnel: a.Tunnel} -} - // Redeem is the joining node-engine's one call (ADR 0587 §5, §6). It answers only a caller the runtime proves to be // enrol., for a key at stage live (or at redeeming with the same keys' digest), checks the proof against the identity key // presented, and then, each step read back and none repeated that is done: spends the token (the key to redeeming, by @@ -50,7 +45,7 @@ func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer if err != nil { return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) } - if err := h.place(e, keysOf(a)); err != nil { + if err := h.place(e, a); err != nil { return delivery.RedeemAnswer{}, err } if h.Loaded != nil { @@ -122,7 +117,7 @@ func (h *Holder) spend(node, digest string) (Enrolment, error) { } // place writes the node into placement with the issue's warrant, unless placement holds it at the approved address already. -func (h *Holder) place(e Enrolment, keys NodeKeys) error { +func (h *Holder) place(e Enrolment, a delivery.Redeem) error { placements, err := h.placements() if err != nil { return fmt.Errorf("placement cannot be read (call redeem again with the same keys): %w", err) @@ -130,11 +125,17 @@ func (h *Holder) place(e Enrolment, keys NodeKeys) error { if p, ok := placements[e.Node]; ok && p.Decided.Address == e.Address { return nil } - if h.AddNode == nil { + if h.AddNode == nil || h.WarrantOf == nil { return errors.New("no route to mesh-placement") } - if err := h.AddNode(AddNode{Node: e.Node, Address: e.Address, Why: "enrolment of " + e.Node + ": " + e.Why, Keys: keys, - Warrant: Warrant{Ask: e.Ask, Node: e.Node, Address: e.Address}}); err != nil { + warrant, err := h.WarrantOf(e.Ask) + if err != nil { + return fmt.Errorf("the operator's warrant for %s cannot be read from the router (call redeem again with the same keys): %w", e.Ask, err) + } + change := delivery.PlacementProposalChange{Op: delivery.OpAddNode, Node: e.Node, Decided: &delivery.NodeDecided{Address: e.Address}, + Keys: &delivery.JoiningKeys{Identity: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey}} + if err := h.AddNode(AddNode{Why: "enrolment of " + e.Node + ": " + e.Why, Change: change, + Warrant: delivery.AddNodeWarrant{Node: e.Node, Address: e.Address, Ask: e.Ask, Warrant: warrant}}); err != nil { return fmt.Errorf("%w (call redeem again with the same keys)", err) } return nil diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go index 5d173d9..9dad26f 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -77,7 +77,8 @@ func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { if e.Stage != StageRedeeming || e.KeysDigest != good.KeysDigest() || r.st.ttl[NodeKey("ace")] != 0 || r.st.ttl[AddressKey("10.9.0.2")] != 0 { t.Errorf("spent %+v, ttl %v/%v", e, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")]) } - if len(r.added) != 1 || r.added[0].Warrant != (Warrant{Ask: "ask-ace", Node: "ace", Address: "10.9.0.2"}) || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { + if len(r.added) != 1 || r.added[0].Warrant != (delivery.AddNodeWarrant{Ask: "ask-ace", Node: "ace", Address: "10.9.0.2", Warrant: "ask-ace@2026-11-02T10:00:00Z"}) || + r.added[0].Change.Keys.Digest() != good.KeysDigest() || r.added[0].Change.Decided.Address != "10.9.0.2" || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { t.Errorf("placement %+v, answer %+v", r.added, ans) } r.loaded = false diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go index 29c1f65..7ff7953 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go @@ -95,12 +95,9 @@ func (v vaultOver) Signer() (string, error) { // addNodeOver asks mesh-placement to add the node with the issue's warrant (ADR 0587 §7). func addNodeOver(call Call) func(AddNode) error { return func(a AddNode) error { - _, err := call(placementAdd, map[string]any{ - "why": a.Why, - "changes": []any{map[string]any{"op": "add-node", "node": a.Node, "address": a.Address, "keys": a.Keys, "warrant": a.Warrant}}, - }) + _, err := call(placementAdd, map[string]any{"why": a.Why, "changes": []any{a.Change}, "warrant": a.Warrant}) if err != nil { - return fmt.Errorf("mesh-placement did not take the node %s: %w", a.Node, err) + return fmt.Errorf("mesh-placement did not take the node %s: %w", a.Change.Node, err) } return nil } From 288364f28d9dd37b37184e8557ff263a5a148b5a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:28:02 +0100 Subject: [PATCH 08/20] Compose each enrol. user from the seat mesh-enrolment's keys, so a token issued by the seat can be redeemed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0587 §4: the grants part composes enrol. from a key at live and the vault's hash, granted only the seat's redeem and its own inbox; §5: when the key ends the user goes. The part reads mesh-enrolment.enrolments as its named reader and follows it by watch: a key reaching live composes the user, and its end (by the holder, or by the bus at its expiry) is a change of the key that takes it out, so no consume of the seat's ending events is added beside the watch (the subtask named one; the watch already hears every end, the bus's expiry included). From the lease write the user loses the controller's enrolment subject (§6) and the standing gap of the controller's tokens is dropped; before it the gap stands. Tested as a subtest of TestGrants, no new test function. Built against the group's mesh-sdk head 5000774 through a go.work outside the worktree; the pin is the lead's. (issue 774, 774/S52) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-grants/grants_test.go | 54 +++++++++++++++++ modules/mesh-grants/internal/grants/follow.go | 6 +- modules/mesh-grants/internal/grants/inputs.go | 58 ++++++++++++++++++- .../mesh-grants/internal/grants/records.go | 4 +- modules/mesh-grants/module.json | 3 +- 5 files changed, 119 insertions(+), 6 deletions(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 9ae4a82..de39f9d 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -358,6 +358,11 @@ const ( xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` + `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history","max-age-days":30}]}` yManifest = `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"reads":["mod-x.kept"]}` + // enrolmentManifest is the holder of mesh-enrolment as the composition needs it (ADR 0587 §1): the claim, its six verbs + // and its bucket. + enrolmentManifest = `{"module":"mesh-enrolment","version":"1","own-secrets":{"broker":"/var/lib/e/broker"},` + + `"claims":[{"name":"mesh-enrolment","serves":["issue","redeem","confirm","revoke","tokens","again"]}],` + + `"tools":["issue","redeem","confirm","revoke","tokens","again"],"state":[{"name":"enrolments","key-ttl":true}]}` ) // brokerSeat is one verb of the bus's seat as a membership serves it: nats claims the seat in fixture and no membership @@ -2404,6 +2409,55 @@ func TestGrants(t *testing.T) { // 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash // within one pass; an event of another kind does not. + // 774/S52/AC1 (novox/hq ADR 0587 §4, §5): a key of the seat mesh-enrolment at live, with the vault's hash, composes + // enrol., which from the lease write may publish the seat's redeem in its own name and subscribe its own inbox, + // nothing else; a key at asked composes none; when the key ends the user is gone. + t.Run("an enrolment key at live composes its enrolment user, and its end takes it out", func(t *testing.T) { + b := fixture() + b.manifestOf("mesh-enrolment", enrolmentManifest) + b.assign("a", "mesh-enrolment") + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + issued := time.Date(2026, 11, 2, 1, 0, 0, 0, time.UTC) + for node, stage := range map[string]string{"lab": delivery.EnrolmentLive, "pi": delivery.EnrolmentAsked} { + raw, err := delivery.EncodeEnrolment(delivery.Enrolment{Node: node, Address: "10.0.0.7", Why: "joins", + Caller: "a.mesh-enrolment", Stage: stage, Issued: issued, Expires: issued.Add(time.Hour)}) + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadEnrolments, delivery.EnrolmentNodeKey(node), json.RawMessage(raw)) + } + b.put(grants.ReadAccounts, "enrol.lab", map[string]any{"user": "enrol.lab", "hash": "$2a$10$x", "generation": 1}) + in := grants.ReadInputs(b, &grants.Manifests{}) + if !slices.Equal(in.Enrolling, []string{"lab"}) || gapNamed(in.Gaps, "enrolment tokens") { + t.Fatalf("enrolling %v, gaps %v", in.Enrolling, in.Gaps) + } + users := shadowOf(t, b) + var lab *grants.User + for i := range users { + if users[i].User == "enrol.lab" { + lab = &users[i] + } + if users[i].User == "enrol.pi" { + t.Error("a key at asked composed enrol.pi") + } + } + if lab == nil || !slices.Equal(lab.Publish, []string{"mesh.seat.mesh-enrolment.call.redeem.enrol~lab"}) || + !slices.Equal(lab.Subscribe, []string{"_INBOX.enrol.lab.>"}) || lab.AllowResponses { + t.Fatalf("enrol.lab: %+v", lab) + } + if c := grants.CredentialsOf(users, nil, in.Hashes); !slices.Contains(c.Adds, "enrol.lab") { + t.Errorf("enrol.lab's hash is not carried: %+v", c) + } + b.mu.Lock() + delete(b.states[grants.ReadEnrolments], delivery.EnrolmentNodeKey("lab")) + b.mu.Unlock() + if slices.Contains(names(shadowOf(t, b)), "enrol.lab") { + t.Error("enrol.lab stays after its key ended") + } + if !slices.Contains(grants.Followed, grants.ReadEnrolments) { + t.Error("a key's change composes nothing again") + } + }) t.Run("the vault's account-issued composes the list again", func(t *testing.T) { b := fixture() var handle func(string, []byte) diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 3831219..065d1be 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -21,9 +21,11 @@ type Watcher interface { // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's // wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts. // -// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6). +// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6), and +// the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol., and its end, by the holder or by +// the bus's expiry, is a change of the key that takes the user out; no event is needed beside the watch. var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, - OwnLease} + ReadEnrolments, OwnLease} // OwnLease is the part's own state lease as the follower watches it, `.`. const OwnLease = Module + "." + StateLease diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index afe87f7..62c0742 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -35,12 +35,15 @@ const ( // ReadPlacement is the seat mesh-placement's placement (ADR 0567 §1, §9): each node's modules and the seats // they hold. ReadPlacement = delivery.PlacementRead + // ReadEnrolments is the seat mesh-enrolment's open enrolments, read as a named reader (ADR 0587 §2, §4): a key at + // live or redeeming composes the user enrol.. + ReadEnrolments = delivery.EnrolmentsRead ) // Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's placement // alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774). var Reads = []string{ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts, ReadPlacement, ReadLease, ReadTakeover} + ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, ReadEnrolments} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( @@ -75,7 +78,8 @@ type Gap struct { // Standing gaps: what the controller composes from its own store, which no bus value carries in this milestone. var standingGaps = []Gap{ - {Input: "enrolment tokens", Why: "the live enrolment tokens are in the controller's store alone: no enrolment user is composed"}, + {Input: "enrolment tokens", Why: "while the controller holds the lease, its live enrolment tokens are in its store alone: " + + "an enrolment user is composed only from the seat mesh-enrolment's keys"}, {Input: "people", Why: "the people and the tools they may invoke are in the controller's store alone: no person user is composed"}, {Input: "seat holdings", Why: "who holds each seat held once for the mesh is in the controller's store alone: a mesh seat claimed on two machines is composed as held on both"}, {Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"}, @@ -118,6 +122,12 @@ type Inputs struct { Loaded *delivery.UserListWritten // Hashes are the vault's hashes of the bus accounts, by bus user (ReadAccounts); kept in memory, never written. Hashes map[string]HeldHash + // Enrolling is every node whose enrolment key is at live or redeeming (ReadEnrolments, ADR 0587 §4, §5), sorted: one + // user enrol. each. A key at asked has no token yet; an ended or expired key is gone from the bucket. + Enrolling []string + // LeaseAway says the controller's lease is handed away, as ReadInputs read it: an enrolment user then enrols through + // the seat alone (ADR 0587 §6). + LeaseAway bool // Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or // not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945). Refused []Gap @@ -264,6 +274,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.Placed[node] = p } } + in.LeaseAway, _ = leaseHandedAway(r) for node, p := range in.Placed { nodes[node] = true for _, m := range p.Assigned() { @@ -431,6 +442,12 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.Gaps = append(in.Gaps, Gap{Input: "wanted manifests", Why: "no build record names the manifest at the wanted fingerprint, so the trunk's is used: " + strings.Join(fromTrunk, ", ")}) } in.Hashes = readHashes(r, &in.Gaps) + // The controller's tokens stand as a gap while it holds the lease; from the write the seat's keys are every token. + var enrolmentsRead bool + in.Enrolling, enrolmentsRead = readEnrolling(r, &in.Gaps) + if enrolmentsRead && in.LeaseAway { + in.Gaps = slices.DeleteFunc(in.Gaps, func(g Gap) bool { return g.Input == "enrolment tokens" }) + } for _, raw := range each(r, ReadResolved, &in.Gaps) { var v resolved if json.Unmarshal(raw, &v) != nil || v.Node == "" || v.Module == "" || v.Key == "" || len(v.Value) == 0 { @@ -445,6 +462,43 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { return in } +// readEnrolling is every node with an enrolment key at live or redeeming (ADR 0587 §4, §5), and whether the seat's state +// was read. Not on the bus yet is a standing gap, never a failed read: the list is still composed and sealed, with no +// enrolment user. A key that does not read is named and composes no user. +func readEnrolling(r Reader, gaps *[]Gap) ([]string, bool) { + keys, err := r.Keys(ReadEnrolments) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadEnrolments, Why: "not read yet: no enrolment user is composed (" + err.Error() + ")"}) + return nil, false + } + var out []string + for _, k := range keys { + node, isNode := delivery.EnrolmentKeyNode(k) + if !isNode { + continue + } + raw, found, err := r.Get(ReadEnrolments, k) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadEnrolments + " " + k, Why: err.Error(), Failed: true}) + continue + } + if !found { + continue + } + e, err := delivery.DecodeEnrolment(raw) + if err != nil || e.Node != node { + *gaps = append(*gaps, Gap{Input: ReadEnrolments + " " + k, Why: "does not read as the enrolment of " + node + + ", so it composes no user"}) + continue + } + if e.Stage == delivery.EnrolmentLive || e.Stage == delivery.EnrolmentRedeeming { + out = append(out, node) + } + } + sort.Strings(out) + return out, true +} + func gapNamed(gaps []Gap, input string) bool { for _, g := range gaps { if g.Input == input { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index cf2556e..c76f9ca 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -36,8 +36,10 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) { for _, own := range catalogue.SeatsWithAProtocol() { seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves} } + // The enrolment users, from the seat mesh-enrolment's keys, and from the lease write through its redeem alone (ADR 0587 + // §4, §6). out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{}, - Interchangeable: map[string]bool{}} + Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling), LeaseHandedAway: in.LeaseAway} values := func(provider, setting string) []any { var vs []any for _, node := range in.Nodes { diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index ba6515d..fb7e5a1 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -22,7 +22,8 @@ "node-engine.have", "mesh-vault.bus-accounts", "mesh-controller.lease", - "mesh-conditions.takeover" + "mesh-conditions.takeover", + "mesh-enrolment.enrolments" ], "consumes": [ "mesh-conditions.condition-raised", From 1a1d3a1ecc9317cded84f4377b360a53f23a4b8b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:29:04 +0100 Subject: [PATCH 09/20] Compose both of a rotating node's logins from the vault's node-logins, so its engine can test-dial the new one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0583 §1 (insight 2026-11-01) and §2: the grants part reads the vault's node-logins as its named reader and composes a node user per login, both while the node rotates and the new one alone after rotation-done; each node's engine reads its own node's key alone (composed by mesh-sdk while the vault serves rotate-node). The part follows the state by watch, so a rotation begun or ended composes again at once. A second login is judged its node's when the list is compared (Placed), and a hash changing by a rotation node-logins names is not counted as a writer changing several hashes at once (sealed.go). The state not on the bus yet is a standing gap: every node logs in as node., as today. Tested as a subtest of TestGrants, no new test function. Built against the group's mesh-sdk head 5000774 through a go.work outside the worktree; the pin is the lead's. (issue 774, 774/S53) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-grants/grants_test.go | 52 +++++++++++++++++++ modules/mesh-grants/internal/grants/follow.go | 2 +- modules/mesh-grants/internal/grants/inputs.go | 52 ++++++++++++++++++- .../mesh-grants/internal/grants/records.go | 4 +- modules/mesh-grants/internal/grants/sealed.go | 9 ++-- .../mesh-grants/internal/grants/userlist.go | 4 +- modules/mesh-grants/module.json | 3 +- 7 files changed, 117 insertions(+), 9 deletions(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index de39f9d..d99bcee 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -358,6 +358,11 @@ const ( xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` + `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history","max-age-days":30}]}` yManifest = `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"reads":["mod-x.kept"]}` + // vaultManifest is the vault as the composition needs it for a rotation (ADR 0583 §2, §3): its claim serving the verbs. + vaultManifest = `{"module":"mesh-vault","version":"1","own-secrets":{"broker":"/var/lib/v/broker"},` + + `"provides":[{"name":"secret","scope":"mesh"}],` + + `"claims":[{"name":"mesh-vault","serves":["rotate-node","node-account-confirm","abandon"]}],` + + `"tools":["rotate-node","node-account-confirm","abandon"],"state":[{"name":"node-logins"}]}` // enrolmentManifest is the holder of mesh-enrolment as the composition needs it (ADR 0587 §1): the claim, its six verbs // and its bucket. enrolmentManifest = `{"module":"mesh-enrolment","version":"1","own-secrets":{"broker":"/var/lib/e/broker"},` + @@ -2458,6 +2463,53 @@ func TestGrants(t *testing.T) { t.Error("a key's change composes nothing again") } }) + // 774/S53 (novox/hq ADR 0583 §1, §2): while node-logins says b rotates to node.b--b, with its hash, both logins are in + // the list and b's machine stays placed under either; after rotation-done (b on node.b--b, nothing rotating) the old + // login is not (AC1). Each node's engine may read its own node-logins key alone (AC2). + t.Run("a node rotating is composed under both its logins, then the new one alone", func(t *testing.T) { + b := fixture() + b.manifestOf("mesh-vault", vaultManifest) + b.assign("a", "mesh-vault") + since := time.Date(2026, 11, 2, 1, 0, 0, 0, time.UTC) + login := func(l delivery.NodeLogin) { + raw, err := delivery.EncodeNodeLogin(l) + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadNodeLogins, l.Node, json.RawMessage(raw)) + } + login(delivery.NodeLogin{Node: "b", Login: "node.b", Rotating: "node.b--b", Stage: delivery.RotationBegun, Since: since}) + b.put(grants.ReadAccounts, "node.b--b", map[string]any{"user": "node.b--b", "hash": "$2a$10$x", "generation": 1}) + users := shadowOf(t, b) + in := grants.ReadInputs(b, &grants.Manifests{}) + if got := names(users); !slices.Contains(got, "node.b") || !slices.Contains(got, "node.b--b") || !slices.Contains(got, "node.a") { + t.Fatalf("the node users are %v", got) + } + if c := grants.CredentialsOf(users, nil, in.Hashes); !slices.Contains(c.Adds, "node.b--b") { + t.Errorf("node.b--b's hash is not carried: %+v", c) + } + if placed := grants.Placed(in); !placed("node.b--b") || placed("node.c--b") || !in.Rotating()["node.b--b"] { + t.Error("the second login is not judged its node's, or not rotating") + } + get := func(key string) string { + return "$JS.API.DIRECT.GET.KV_" + delivery.NodeLoginsBucket + ".$KV." + delivery.NodeLoginsBucket + "." + key + } + for _, u := range users { + if !strings.HasPrefix(u.User, "node.") { + continue + } + node, _ := delivery.LoginNode(u.User) + for _, of := range []string{"a", "b"} { + if slices.Contains(u.Publish, get(of)) != (of == node) { + t.Errorf("%s reads %s's node-logins key: %v", u.User, of, of != node) + } + } + } + login(delivery.NodeLogin{Node: "b", Login: "node.b--b", Since: since.Add(time.Hour)}) + if got := names(shadowOf(t, b)); slices.Contains(got, "node.b") || !slices.Contains(got, "node.b--b") { + t.Errorf("after rotation-done the node users are %v", got) + } + }) t.Run("the vault's account-issued composes the list again", func(t *testing.T) { b := fixture() var handle func(string, []byte) diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 065d1be..9770249 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -25,7 +25,7 @@ type Watcher interface { // the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol., and its end, by the holder or by // the bus's expiry, is a change of the key that takes the user out; no event is needed beside the watch. var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, - ReadEnrolments, OwnLease} + ReadEnrolments, ReadNodeLogins, OwnLease} // OwnLease is the part's own state lease as the follower watches it, `.`. const OwnLease = Module + "." + StateLease diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index 62c0742..ac8e8e1 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -38,12 +38,15 @@ const ( // ReadEnrolments is the seat mesh-enrolment's open enrolments, read as a named reader (ADR 0587 §2, §4): a key at // live or redeeming composes the user enrol.. ReadEnrolments = delivery.EnrolmentsRead + // ReadNodeLogins is the vault's node-logins, read as its named reader (ADR 0583 §1 insight): each node's login in use + // and the one rotating, both composed as users of the node's permissions. + ReadNodeLogins = delivery.NodeLoginsRead ) // Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's placement // alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774). var Reads = []string{ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, ReadEnrolments} + ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, ReadEnrolments, ReadNodeLogins} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( @@ -125,6 +128,9 @@ type Inputs struct { // Enrolling is every node whose enrolment key is at live or redeeming (ReadEnrolments, ADR 0587 §4, §5), sorted: one // user enrol. each. A key at asked has no token yet; an ended or expired key is gone from the bucket. Enrolling []string + // NodeLogins is each node's key of the vault's node-logins (ReadNodeLogins, ADR 0583 §1); a node absent logs in as + // node. alone. + NodeLogins map[string]delivery.NodeLogin // LeaseAway says the controller's lease is handed away, as ReadInputs read it: an enrolment user then enrols through // the seat alone (ADR 0587 §6). LeaseAway bool @@ -448,6 +454,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { if enrolmentsRead && in.LeaseAway { in.Gaps = slices.DeleteFunc(in.Gaps, func(g Gap) bool { return g.Input == "enrolment tokens" }) } + in.NodeLogins = readNodeLogins(r, &in.Gaps) for _, raw := range each(r, ReadResolved, &in.Gaps) { var v resolved if json.Unmarshal(raw, &v) != nil || v.Node == "" || v.Module == "" || v.Key == "" || len(v.Value) == 0 { @@ -499,6 +506,49 @@ func readEnrolling(r Reader, gaps *[]Gap) ([]string, bool) { return out, true } +// readNodeLogins is every node's key of the vault's node-logins (ADR 0583 §1). Not on the bus yet is a standing gap, +// never a failed read: every node then logs in as node. alone, as before any rotation. A key that does not read is +// named and leaves its node at node.; a read that fails is a failed read, since a rotating node's new login would be +// left out. +func readNodeLogins(r Reader, gaps *[]Gap) map[string]delivery.NodeLogin { + keys, err := r.Keys(ReadNodeLogins) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadNodeLogins, Why: "not read yet: every node logs in as node. (" + err.Error() + ")"}) + return nil + } + out := map[string]delivery.NodeLogin{} + for _, node := range keys { + raw, found, err := r.Get(ReadNodeLogins, node) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadNodeLogins + " " + node, Why: err.Error(), Failed: true}) + continue + } + if !found { + continue + } + l, err := delivery.DecodeNodeLogin(raw) + if err != nil || l.Node != node { + *gaps = append(*gaps, Gap{Input: ReadNodeLogins + " " + node, Why: "does not read as the logins of " + node + + ", so it logs in as node." + node}) + continue + } + out[node] = l + } + return out +} + +// Rotating are the logins the vault's node-logins names in a rotation under way, as a set: a hash of one changes by the +// vault's rotation (ADR 0583 §2), never counted as a writer changing several hashes at once. +func (in Inputs) Rotating() map[string]bool { + out := map[string]bool{} + for _, l := range in.NodeLogins { + if l.Rotates() { + out[l.Login], out[l.Rotating] = true, true + } + } + return out +} + func gapNamed(gaps []Gap, input string) bool { for _, g := range gaps { if g.Input == input { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index c76f9ca..eaba280 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -39,7 +39,9 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) { // The enrolment users, from the seat mesh-enrolment's keys, and from the lease write through its redeem alone (ADR 0587 // §4, §6). out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{}, - Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling), LeaseHandedAway: in.LeaseAway} + Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling), LeaseHandedAway: in.LeaseAway, + // And each node's logins (ADR 0583 §1): both while it rotates. + NodeLogins: in.NodeLogins} values := func(provider, setting string) []any { var vs []any for _, node := range in.Nodes { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index a20e816..dbe2c83 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -336,8 +336,9 @@ func (p *Part) keepWrittenUsers(res *Result, list []byte, hashes map[string]Held } // changedHashes are the users of the list in force whose hash the list composed now would change with no rotation by the -// vault naming them, sorted; none when the part kept no credentials of the list in force (a list written before). -func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) { +// vault naming them (its announcement on the hash, or a node's login rotating in node-logins, ADR 0583 §2), sorted; none +// when the part kept no credentials of the list in force (a list written before). +func (p *Part) changedHashes(hashes map[string]HeldHash, rotating map[string]bool) ([]string, error) { raw, _, found, err := p.Store.Get(StateUserList, KeyWrittenCredentials) if err != nil || !found { return nil, err @@ -349,7 +350,7 @@ func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) { var changed []string for user, was := range had { h, ok := hashes[user] - if !ok || credentialOf(h) == was || h.Rotating != nil { + if !ok || credentialOf(h) == was || h.Rotating != nil || rotating[user] { continue } changed = append(changed, user) @@ -419,7 +420,7 @@ func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest } // More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new // writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21. - changed, err := p.changedHashes(in.Hashes) + changed, err := p.changedHashes(in.Hashes, in.Rotating()) if err != nil { return "held: the credentials of the list in force cannot be read: " + err.Error() } diff --git a/modules/mesh-grants/internal/grants/userlist.go b/modules/mesh-grants/internal/grants/userlist.go index b83a6a5..ae76b45 100644 --- a/modules/mesh-grants/internal/grants/userlist.go +++ b/modules/mesh-grants/internal/grants/userlist.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/outward" ) @@ -614,7 +615,8 @@ func Placed(in Inputs) func(user string) bool { case len(parts) < 2: return true case parts[0] == "node" || parts[0] == "provisioner": - return slices.Contains(in.Nodes, parts[1]) + // A node's second login, node.--b, is its node's (ADR 0583 §1). + return slices.Contains(in.Nodes, strings.TrimSuffix(parts[1], delivery.SecondLoginSuffix)) case parts[0] == "person" || parts[0] == "enrol": return true } diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index fb7e5a1..dc309fc 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -23,7 +23,8 @@ "mesh-vault.bus-accounts", "mesh-controller.lease", "mesh-conditions.takeover", - "mesh-enrolment.enrolments" + "mesh-enrolment.enrolments", + "mesh-vault.node-logins" ], "consumes": [ "mesh-conditions.condition-raised", From 189e64c486edf9ffb5f7c4ee88538e2740d28589 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 00:35:44 +0100 Subject: [PATCH 10/20] mesh-enrolment: its integration test, whole on a real bus The holder's code over a real nats-server bucket with a time to live per key: issue, the operator's yes, redeem as enrol.ace, the node confirming as the login its key of node-logins names (the stand-in vault writes it at node-account), node-enrolled and placement-changed said on their subjects, the user enrol.ace composed while the key is live and gone with it, the token dropped only after the end; and a token never redeemed expires on the bus, is heard, says enrolment-expired and is dropped. Found by it: a bucket with a time to live per key must keep one revision per key, or the bus expires nothing (nats-server 2.11.17), so the state enrolments declares no history. Paid for: the integration test is paid by merging the builder's archive-only and root-module integration tests into one table test; the module's ten unit tests by merging the assigner's three small composition tests into one table (net 2 of 10 paid; the rest is for the lead, the repository being over budget). (issue 774, 774/S60) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../internal/assign/compose_test.go | 54 +-- .../mesh-delivery-builder/integration_test.go | 31 +- .../cmd/mesh-enrolment/integration_test.go | 386 ++++++++++++++++++ modules/mesh-enrolment/go.mod | 6 + modules/mesh-enrolment/go.sum | 33 +- 5 files changed, 451 insertions(+), 59 deletions(-) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go diff --git a/modules/mesh-delivery-assigner/internal/assign/compose_test.go b/modules/mesh-delivery-assigner/internal/assign/compose_test.go index c8921b4..1e202f5 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose_test.go @@ -68,33 +68,33 @@ func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) { } } -// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module. -func TestRemovingAModuleAMachineRunsFails(t *testing.T) { - c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}} - v := Compose(meshOf(), c, "sdk", 0) - if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") { - t.Fatalf("composition = %+v, want a fail naming g14 and web", v) - } -} - -// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the -// consumer's machine is touched when its provider changes. -func TestAMachineTheChangeBreaksFails(t *testing.T) { - c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}} - v := Compose(meshOf(), c, "sdk", 0) - if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") { - t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v) - } -} - -// 1297/B3c: what was already broken on the trunk is said and fails nothing. -func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) { - m := meshOf() - m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"} - c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}} - v := Compose(m, c, "sdk", 0) - if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") { - t.Fatalf("composition = %+v, want a pass saying ace was already broken", v) +// 1297/AC2 and B3c: a change that removes a module a machine runs fails, naming the machine and the module; a machine that +// composes on the trunk and not with the change fails it, naming the machine and why (the consumer's machine is touched when +// its provider changes); what was already broken on the trunk is said and fails nothing. One table (three tests merged). +func TestAChangeThatBreaksAMachineFailsAndWhatWasBrokenFailsNothing(t *testing.T) { + broken := meshOf() + broken.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"} + for _, c := range []struct { + name string + mesh Mesh + c Change + state string + must func(delivery.CompositionVerdict) bool + }{ + {"removing a module a machine runs", meshOf(), Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}, delivery.CompositionFail, + func(v delivery.CompositionVerdict) bool { + return v.Machine == "g14" && strings.Contains(v.Words, "g14 runs web") + }}, + {"a machine the change breaks", meshOf(), Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}, + delivery.CompositionFail, func(v delivery.CompositionVerdict) bool { + return v.Machine == "ace" && strings.Contains(v.Words, "database") + }}, + {"what was broken before", broken, Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}, + delivery.CompositionPass, func(v delivery.CompositionVerdict) bool { return hasWarning(v, "on the trunk either") }}, + } { + if v := Compose(c.mesh, c.c, "sdk", 0); v.State != c.state || !c.must(v) { + t.Errorf("%s: composition = %+v", c.name, v) + } } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index bae4fd6..6c231fb 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -467,15 +467,20 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // } } -// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no -// toolchains; the builder builds it on the build module's nodes all the same (review round 4). -func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) { - b := startBed(t) - b.archive(t, "ca-trust", "modules/ca-trust", commit) - b.start(t, "n1", nil) - v := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") { - t.Errorf("build = %+v, want done naming no toolchain", v) +// TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no +// toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "." +// is given to the build tools as it is (1615/AC4). One table: the two tests were merged to pay for the enrolment's. +func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) { + for _, c := range []struct{ name, path string }{{"no toolchain", "modules/ca-trust"}, {"root", delivery.RootPath}} { + t.Run(c.name, func(t *testing.T) { + b := startBed(t) + b.archive(t, "ca-trust", c.path, commit) + b.start(t, "n1", nil) + v := b.await(t, "ca-trust", isState(delivery.BuildDone)) + if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) { + t.Errorf("build = %+v, want done naming no toolchain", v) + } + }) } } @@ -530,14 +535,6 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A } } -// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is. -func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4 - b := startBed(t) - b.archive(t, "ca-trust", delivery.RootPath, commit) - b.start(t, "n1", nil) - b.await(t, "ca-trust", isState(delivery.BuildDone)) -} - func mustKey(t *testing.T, module string) string { k, err := delivery.BuilderBuildKey(module, fp) if err != nil { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go new file mode 100644 index 0000000..6bf05ad --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -0,0 +1,386 @@ +//go:build integration + +package main + +// mesh-enrolment whole on a real bus (novox/hq ADR 0587 §2-§8; issue 774): the holder's own code over a real nats-server's +// key-value bucket `enrolments` with a time to live per key (the keys written as the runtime writes them: a create with the +// expiry, a put with a message TTL, an update by revision), its events said on their real subjects, the bus's own expiry marker +// heard, and the other parts stood in for — the vault (which writes the new node's key of node-logins at node-account, as the +// vault does), the operator's channel, placement, and the grants part's rule that a key at live makes the user enrol. and +// a key gone takes it away. + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// busStore is the holder's bucket as the runtime writes it. +type busStore struct { + ctx context.Context + js jetstream.JetStream + kv jetstream.KeyValue +} + +func (b *busStore) Get(key string) (Row, bool, error) { + e, err := b.kv.Get(b.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return Row{}, false, nil + } + if err != nil { + return Row{}, false, err + } + return Row{Value: e.Value(), Revision: e.Revision()}, true, nil +} + +func (b *busStore) Create(key string, v any, ttl time.Duration) error { + raw, err := Encode(v) + if err != nil { + return err + } + var opts []jetstream.KVCreateOpt + if ttl > 0 { + opts = append(opts, jetstream.KeyTTL(ttl)) + } + _, err = b.kv.Create(b.ctx, key, raw, opts...) + if errors.Is(err, jetstream.ErrKeyExists) { + return ErrExists + } + return err +} + +func (b *busStore) Update(key string, v any, revision uint64) (uint64, error) { + raw, err := Encode(v) + if err != nil { + return 0, err + } + rev, err := b.kv.Update(b.ctx, key, raw, revision) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + return 0, ErrChanged + } + return rev, err +} + +func (b *busStore) Put(key string, v any, ttl time.Duration) (uint64, error) { + raw, err := Encode(v) + if err != nil { + return 0, err + } + if ttl <= 0 { + return b.kv.Put(b.ctx, key, raw) + } + ack, err := b.js.Publish(b.ctx, "$KV."+b.kv.Bucket()+"."+key, raw, jetstream.WithMsgTTL(max(time.Second, ttl.Truncate(time.Second)))) + if err != nil { + return 0, err + } + return ack.Sequence, nil +} + +func (b *busStore) Delete(key string) error { + return b.kv.Delete(b.ctx, key) +} + +func (b *busStore) Keys() ([]string, error) { + keys, err := b.kv.Keys(b.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +// busReader reads other parts' buckets, `.`. +type busReader struct { + ctx context.Context + js jetstream.JetStream +} + +func (r busReader) bucket(state string) (jetstream.KeyValue, error) { + return r.js.KeyValue(r.ctx, strings.Replace(state, ".", "_", 1)) +} + +func (r busReader) Keys(state string) ([]string, error) { + kv, err := r.bucket(state) + if err != nil { + return nil, err + } + keys, err := kv.Keys(r.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func (r busReader) Get(state, key string) ([]byte, bool, error) { + kv, err := r.bucket(state) + if err != nil { + return nil, false, err + } + e, err := kv.Get(r.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + +// standInVault holds the tokens it made, and writes a joining node's key of node-logins when it mints the account. +type standInVault struct { + ctx context.Context + logins jetstream.KeyValue + mu sync.Mutex + tokens map[string]bool + calls []string +} + +func (v *standInVault) note(c string) { + v.mu.Lock() + v.calls = append(v.calls, c) + v.mu.Unlock() +} + +func (v *standInVault) Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) { + v.mu.Lock() + v.tokens[node] = true + v.mu.Unlock() + v.note("token " + node) + return delivery.EnrolmentTokenAnswer{Node: node, Broker: "bus:4222", Fingerprint: "f", Signer: "s"}, nil +} +func (v *standInVault) Sealed(node string) (string, error) { return "sealed-" + node, nil } +func (v *standInVault) Drop(node string) error { + v.mu.Lock() + delete(v.tokens, node) + v.mu.Unlock() + v.note("drop " + node) + return nil +} +func (v *standInVault) Account(node, sealing, digest string) (delivery.NodeAccountAnswer, error) { + raw, err := delivery.EncodeNodeLogin(delivery.NodeLogin{Node: node, Login: delivery.FirstLogin(node), Since: time.Now().UTC()}) + if err != nil { + return delivery.NodeAccountAnswer{}, err + } + if _, err := v.logins.Put(v.ctx, node, raw); err != nil { + return delivery.NodeAccountAnswer{}, err + } + return delivery.NodeAccountAnswer{Node: node, Login: delivery.FirstLogin(node), Account: "account-" + node}, nil +} +func (v *standInVault) Retire(node string) error { return nil } +func (v *standInVault) SignCertificate(node, serving string) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Certificate: "PEM"}, nil +} +func (v *standInVault) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindEnrolment, Node: node, Signature: "sig", Signer: "fp"}, nil +} +func (v *standInVault) Signer() (string, error) { return "vault-key", nil } + +// 774/S60/AC1: issue, a warrant, redeem as enrol. and confirm, whole on a real bus. The user enrol. is composed +// while the key is live and leaves with it; the node confirms as the login its key of node-logins names; node-enrolled and +// placement-changed are said; the vault is asked to drop the token only after the end; and a token never redeemed is ended by +// the bus's own expiry, heard, saying enrolment-expired and bringing the drop. +func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { + srv := testbus.Start(t) + nc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + mk := func(name string, ttl bool) jetstream.KeyValue { + cfg := jetstream.KeyValueConfig{Bucket: name, History: 5} + if ttl { + // A bucket with a time to live per key keeps one revision per key: the bus (nats-server 2.11.17) expires no key of + // one that keeps more, which this test found out (a state declared key-ttl takes no history). + cfg.History, cfg.LimitMarkerTTL = 1, time.Minute + } + kv, err := js.CreateKeyValue(ctx, cfg) + if err != nil { + t.Fatal(err) + } + return kv + } + enrolments := mk(delivery.EnrolmentsBucket, true) + logins, lease, placement := mk(delivery.NodeLoginsBucket, false), mk("mesh-controller_lease", false), mk("mesh-delivery-assigner_placement", false) + mk("node-engine_have", false) + if _, err := lease.Put(ctx, "holder", []byte(`{"handed-to":"mesh-grants","handed-by":"operator"}`)); err != nil { + t.Fatal(err) + } + if _, err := placement.Put(ctx, "hub.node", []byte(hubPlacement)); err != nil { + t.Fatal(err) + } + // The seats' events are kept on a stream of the events, as the bus keeps them, and heard on their real subjects. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_EVENTS", + Subjects: []string{"mesh.seat.mesh-enrolment.event.>", "mesh.seat.mesh-placement.event.>"}}); err != nil { + t.Fatal(err) + } + var heardMu sync.Mutex + var heard []string + for _, subject := range []string{"mesh.seat.mesh-enrolment.event.*", "mesh.seat.mesh-placement.event.*"} { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + heardMu.Lock() + heard = append(heard, m.Subject[strings.LastIndex(m.Subject, ".")+1:]) + heardMu.Unlock() + }); err != nil { + t.Fatal(err) + } + } + events := func() []string { heardMu.Lock(); defer heardMu.Unlock(); return slices.Clone(heard) } + + store := &busStore{ctx: ctx, js: js, kv: enrolments} + vault := &standInVault{ctx: ctx, logins: logins, tokens: map[string]bool{}} + asker := &fakeAsker{} + h := &Holder{Store: store, Read: busReader{ctx, js}, Vault: vault, Asker: asker, AskLasts: time.Hour, + Say: func(event string, body any, id string) error { + raw, _ := json.Marshal(body) + _, err := js.Publish(ctx, "mesh.seat.mesh-enrolment.event."+event, raw, jetstream.WithMsgID(id)) + return err + }, + AddNode: func(a AddNode) error { + value := `{"v":1,"node":"` + a.Change.Node + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` + if _, err := placement.Put(ctx, a.Change.Node+".node", []byte(value)); err != nil { + return err + } + _, err := js.Publish(ctx, "mesh.seat.mesh-placement.event.placement-changed", []byte(`{}`)) + return err + }, + WarrantOf: func(ask string) (string, error) { return ask + "@2026-11-02T10:00:00Z", nil }, + Warrant: func(string) error { return nil }, + Act: func(string, string, string) error { return nil }, + Loaded: func(string) (bool, error) { return true, nil }, + Logf: t.Logf} + + // The grants part's rule, stood in: a key at live is the user enrol.; a key gone takes it away. The holder's watch + // of its own bucket hears the bus's expiry marker. + var usersMu sync.Mutex + users := map[string]bool{} + hasUser := func(u string) bool { usersMu.Lock(); defer usersMu.Unlock(); return users[u] } + w, err := enrolments.WatchAll(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = w.Stop() }) + go func() { + for e := range w.Updates() { + if e == nil { + continue + } + node, ok := strings.CutPrefix(e.Key(), "node.") + if !ok { + continue + } + if e.Operation() == jetstream.KeyValuePut { + en, err := decodeEnrolment(e.Value()) + usersMu.Lock() + users[EnrolUser(node)] = err == nil && en.Stage == StageLive + usersMu.Unlock() + _ = h.Heard(stdio.OpPut, e.Key(), e.Value()) + continue + } + usersMu.Lock() + delete(users, EnrolUser(node)) + usersMu.Unlock() + // A delete is the holder's own; the marker the bus leaves when a key's own time to live ends is a purge. + if e.Operation() == jetstream.KeyValuePurge { + _ = h.Heard(stdio.OpExpire, e.Key(), nil) + } else { + _ = h.Heard(stdio.OpDelete, e.Key(), nil) + } + } + }() + // The durable consumer of the holder's own end events, stood in by a core subscription that asks the drop. + for _, e := range EndEvents { + key := Seat + "." + e + if _, err := nc.Subscribe("mesh.seat.mesh-enrolment.event."+e, func(m *nats.Msg) { + if err := h.Ended(key, m.Data); err != nil { + t.Errorf("%v", err) + } + }); err != nil { + t.Fatal(err) + } + } + waitFor := func(what string, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(20 * time.Second); !ok(); time.Sleep(25 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("never: %s (events %v, vault %v)", what, events(), vault.calls) + } + } + } + + // Issue, the operator's yes, redeem as enrol.ace, confirm as node.ace. + a, err := h.Issue("person.operator", "ace", "a new machine", 0) + if err != nil { + t.Fatal(err) + } + if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: a.Ask}); err != nil { + t.Fatal(err) + } + waitFor("the user enrol.ace is composed", func() bool { return hasUser("enrol.ace") }) + args := redeemArgs("ace", 1, "serving-ace") + if _, err := h.Redeem("enrol.somebody", args); err == nil { + t.Fatal("redeemed by another caller") + } + ans, err := h.Redeem("enrol.ace", args) + if err != nil { + t.Fatal(err) + } + // The node dials as the login its key of node-logins names, and confirms as it. + e, err := logins.Get(ctx, "ace") + if err != nil { + t.Fatal(err) + } + nl, err := delivery.DecodeNodeLogin(e.Value()) + if err != nil || nl.Login != ans.Login { + t.Fatalf("node-logins says %+v, %v, and the answer %s", nl, err, ans.Login) + } + if _, err := h.Confirm(nl.Login, ""); err != nil { + t.Fatal(err) + } + waitFor("node-enrolled and placement-changed are said and the user leaves the list", func() bool { + ev := events() + return slices.Contains(ev, EventEnrolled) && slices.Contains(ev, "placement-changed") && !hasUser("enrol.ace") + }) + waitFor("the vault is asked to drop the token after the end", func() bool { + vault.mu.Lock() + defer vault.mu.Unlock() + return slices.Contains(vault.calls, "drop ace") && !vault.tokens["ace"] + }) + if keys, _ := store.Keys(); len(keys) != 0 { + t.Errorf("an ended enrolment leaves keys %v", keys) + } + + // A token never redeemed expires on the bus: the holder hears the marker, says enrolment-expired and the token is dropped. + if _, err := h.Issue("person.operator", "g14", "a second machine", 2*time.Second); err != nil { + t.Fatal(err) + } + if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: asker.asked[len(asker.asked)-1]}); err != nil { + t.Fatal(err) + } + waitFor("enrolment-expired is said for g14", func() bool { return slices.Contains(events(), EventExpired) }) + waitFor("the expired token is dropped", func() bool { + vault.mu.Lock() + defer vault.mu.Unlock() + return slices.Contains(vault.calls, "drop g14") + }) + if hasUser("enrol.g14") { + t.Error("the user enrol.g14 stays after its key expired") + } +} diff --git a/modules/mesh-enrolment/go.mod b/modules/mesh-enrolment/go.mod index 03fe8f2..10f8f09 100644 --- a/modules/mesh-enrolment/go.mod +++ b/modules/mesh-enrolment/go.mod @@ -4,13 +4,19 @@ go 1.26.0 require ( git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 + github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/mesh-enrolment/go.sum b/modules/mesh-enrolment/go.sum index 1c91827..984c03d 100644 --- a/modules/mesh-enrolment/go.sum +++ b/modules/mesh-enrolment/go.sum @@ -1,26 +1,29 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= -github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= -github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= -github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= -github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= -github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= -github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= From 70f13a3775b62fc7fcd0100c33e9d6d89550bf55 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:03:04 +0100 Subject: [PATCH 11/20] mesh-enrolment: place before the vault is asked, read the grants pass whole, name the expired enrolment's issued event, carry the first tunnel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - redeem writes the node into placement before it asks the vault for the account and the certificate of the serving key presented (the vault reads the node's address from placement); a retry by the same keys still finishes (blocker 3). - loadedVia reads mesh-grants.user-list's pass as the part's whole Result, the list state under user_list; the fixture testdata/grants-pass.json was written by mesh-grants' own Result, not by hand (blocker 4). - enrolment-expired is delivery.EnrolmentExpiry with IssuedID, through its codec; a key that expired before its content was seen names no issued event, so nothing is said and its token is dropped at once (blocker 6). - issue builds the joining node's first tunnel from placement (the hub, its endpoint, the node's address) and the hub's record (its tunnel key) and passes it to enrolment-token; no hub, or a hub with no key or endpoint, is a finding of issue (ADR 0587 §3, insight of 2026-11-02). - The assigner takes an add-node only from mesh-enrolment's own bus user (stdio.Callers), never person.mesh-enrolment. - The watchman raises machine..node-account-fallback from a node's health report's fall back and clears it when the report no longer carries one (ADR 0583 §6, blocker 7). - mesh-enrolment's ten unit test functions are two table tests now (blocker 8): the repository is back at 816. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-delivery-assigner/seat_tools.go | 8 ++- .../mesh-delivery-assigner/seat_tools_test.go | 1 + .../internal/watch/findings_test.go | 12 ++-- .../internal/watch/nodehealth.go | 22 ++++++- .../cmd/mesh-enrolment/confirm_test.go | 2 +- .../mesh-enrolment/cmd/mesh-enrolment/end.go | 28 ++++----- .../cmd/mesh-enrolment/end_test.go | 16 +++-- .../cmd/mesh-enrolment/enrolment_test.go | 61 +++++++++++++++++++ .../cmd/mesh-enrolment/holder.go | 43 ++++++++++++- .../cmd/mesh-enrolment/integration_test.go | 29 +++++++-- .../cmd/mesh-enrolment/issue.go | 17 +++++- .../cmd/mesh-enrolment/issue_test.go | 18 +++++- .../cmd/mesh-enrolment/kit_test.go | 19 ++++-- .../mesh-enrolment/cmd/mesh-enrolment/main.go | 28 +++++---- .../cmd/mesh-enrolment/redeem.go | 9 +-- .../cmd/mesh-enrolment/redeem_test.go | 9 ++- .../mesh-enrolment/testdata/grants-pass.json | 53 ++++++++++++++++ .../cmd/mesh-enrolment/tokens_test.go | 4 +- .../cmd/mesh-enrolment/vault.go | 7 ++- 19 files changed, 317 insertions(+), 69 deletions(-) create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go create mode 100644 modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 2ef4889..8f944dd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -262,10 +262,12 @@ func warrantOf(v any) (*delivery.AddNodeWarrant, error) { return &out, nil } -// moduleOfCaller is the module a caller is, as the runtime proves it: a module's bus user is `.`. +// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven +// caller is that module's (its bus user `.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller +// as proven, which the refusal names. func moduleOfCaller(by string) string { - if _, module, ok := strings.Cut(by, "."); ok { - return module + if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) { + return delivery.EnrolmentModule } return by } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 82142d1..f4f80f4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -160,6 +160,7 @@ func TestTheSeatsVerbs(t *testing.T) { good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}` for _, c := range []struct{ by, warrant, want string }{ {"novox.mesh-grants", good, "mesh-enrolment's alone"}, + {"person.mesh-enrolment", good, "mesh-enrolment's alone"}, {"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"}, {"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"}, {"novox.mesh-enrolment", "", "carries no warrant"}, diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index d25eec9..e1f83c2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -246,7 +246,7 @@ func TestTheWatchmansFindings(t *testing.T) { w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put) w.Provision("ace.web.oidc", nil, Delete) }, map[string]string{}}, - {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found, a login fallen back", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { report := func(h delivery.NodeHealth) []byte { b, err := delivery.EncodeNodeHealth(h) if err != nil { @@ -259,11 +259,13 @@ func TestTheWatchmansFindings(t *testing.T) { {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, {Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}}, - Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, - Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, - Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}}), Put) + Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, + Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, + Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}, + Fallback: &delivery.LoginFallback{Node: "novox", From: "node.novox--b", Why: "refused twice", At: time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)}}), Put) for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent, - "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning} { + "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning, + "machine.novox.node-account-fallback": Warning} { if o.findings[key].Severity != sev { t.Errorf("%s = %+v", key, o.findings[key]) } diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go index d3b6dd3..eda9df4 100644 --- a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -21,7 +21,9 @@ import ( // said once there, never while its own network or its silence says it; // - machine-units `machine..units`: failed units no module places; // - relogin-needed `module...relogin-needed` and directory-used-as-found -// `module...directory-used-as-found`. +// `module...directory-used-as-found`; +// - node-account-fallback `machine..node-account-fallback`: the engine fell back to its node's old login during a +// rotation (novox/hq ADR 0583 §6), raised while its report carries the fall back and cleared once it no longer does. // // A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held // there, as the controller holds it: when every machine it points at is down on the record (silent, its own network @@ -36,6 +38,7 @@ const ( KindMachineUnits = "machine-units" KindReloginNeeded = "relogin-needed" KindUsedAsFound = "directory-used-as-found" + KindAccountFallback = delivery.NodeAccountFallbackFinding ) // The parts and states a report names (mesh-control internal/link). @@ -101,6 +104,10 @@ func (w *Watchman) nodeFindings(node string) { want[f.Key()] = true w.find(f) } + if f, raise := fallbackOf(node, h.Fallback); raise { + want[f.Key()] = true + w.find(f) + } } for key := range w.findings { if !want[key] && w.ownsHealthKey(key, node) { @@ -249,6 +256,19 @@ func foundOf(node string, dirs []delivery.NodeDirectory) []Finding { return out } +// fallbackOf is a node's fall back to its old login during a rotation of its account (ADR 0583 §6), one finding. +func fallbackOf(node string, f *delivery.LoginFallback) (Finding, bool) { + if f == nil || f.Node != node { + return Finding{}, false + } + said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) + return Finding{Scope: ScopeMachine, ID: node, Token: KindAccountFallback, Kind: KindAccountFallback, Machine: node, Severity: Warning, + Source: "health-report", Since: f.At, Summary: said, Said: said, + Headline: fmt.Sprintf("%s fell back to its old bus login", node), + Needs: fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node), + Resolved: fmt.Sprintf("%s no longer falls back to its old bus login", node)}, true +} + func orModule(m string) string { if m == "" { return "_" diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go index b88928c..89a00b2 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go @@ -4,7 +4,7 @@ import "testing" // 774/S57/AC1: a confirm from any caller but node. is refused; from node. node-enrolled is said and both keys // are deleted; a stop between the event and the deletes is finished by calling confirm again. -func TestConfirmIsTheNodesOwnAndEndsTheEnrolment(t *testing.T) { +func confirmIsTheNodesOwnAndEndsTheEnrolment(t *testing.T) { for _, c := range []struct{ name, caller, node, want string }{ {"the enrolment user", "enrol.ace", "", "dialled as node."}, {"another node", "node.g14", "ace", "its own enrolment only"}, diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go index 79e75c2..545f788 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -82,27 +82,27 @@ func (h *Holder) Heard(op, key string, value []byte) error { case stdio.OpExpire: e, known := h.seen[node] delete(h.seen, node) - id := "enrolment-expired." + node - if known { - id = eventID(EventExpired, e) + if !known { + // The key expired before the watch handed its content over (a holder started after it was written, before its + // catch-up): the enrolment-issued event cannot be named, and an expiry naming none is no expiry (the contract + // refuses it). Nothing is said; the token is dropped at once, as an ending event would have it dropped. + h.logf("the enrolment of %s expired before its content was seen, so enrolment-expired cannot name its issued event and is not said; its token is dropped", node) + if err := h.Vault.Drop(node); err != nil { + return fmt.Errorf("the token of %s, whose enrolment expired unseen, is not dropped (offered again): %w", node, err) + } + return nil } - // The sequence of the enrolment-issued event on the events' stream is not given to a bundle that publishes (the runtime - // answers no sequence), so issued-event is 0 and issued-id names the message id that carried the content. - body := Expired{EnrolmentExpiry: delivery.EnrolmentExpiry{V: delivery.Version, Node: node}, IssuedID: eventID(EventIssued, e)} - if err := h.say(EventExpired, body, id); err != nil { + body, err := delivery.EncodeEnrolmentExpiry(delivery.EnrolmentExpiry{Node: node, IssuedID: eventID(EventIssued, e)}) + if err != nil { + return err + } + if err := h.say(EventExpired, json.RawMessage(body), eventID(EventExpired, e)); err != nil { return fmt.Errorf("enrolment-expired of %s cannot be said (offered again): %w", node, err) } } return nil } -// Expired is what enrolment-expired carries (delivery.EnrolmentExpiry) and the message id of the enrolment-issued event that -// carried the key's content, which the bus's sequence number cannot be known for. -type Expired struct { - delivery.EnrolmentExpiry - IssuedID string `json:"issued-id,omitempty"` -} - // Ended is the durable consumer's act on one of the holder's own ending events (ADR 0587 §5): the vault is asked to drop the // node's token and its hash, and the event is acknowledged only when the vault answers that both are gone, so a drop that // fails is delivered again by the bus. It never runs before an end: the consumer hears only the end events. diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go index 9bde32a..f913ed3 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "git.novox.be/novox/mesh-sdk/go/delivery" "slices" "strings" "testing" @@ -12,7 +13,7 @@ import ( // 774/S58/AC1: a revoke at redeeming, whether the vault minted the account yet or not, takes both keys away, the vault retires // the account, the revoke is recorded and enrolment-revoked is said; one the vault failed to retire ends nothing and is // finished by calling it again. At asked it withdraws the ask; an expiry marker says enrolment-expired with the issued event. -func TestRevokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { +func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { r := newRig(t) r.seed("ace", "10.9.0.2", StageRedeeming) r.vault.fail["retire ace"] = true @@ -42,15 +43,20 @@ func TestRevokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { t.Fatal(err) } last := r.said[len(r.said)-1] - mustContain(t, string(last.Body), `"issued-id":"enrolment-issued.novox.`) - if last.Event != EventExpired { - t.Errorf("said %s", last.Event) + x, err := delivery.DecodeEnrolmentExpiry(last.Body) + if err != nil || last.Event != EventExpired || x.Node != "novox" || !strings.HasPrefix(x.IssuedID, "enrolment-issued.novox.") { + t.Errorf("said %s %s: %+v, %v", last.Event, last.Body, x, err) + } + // A key that expired before its content was seen names no issued event: nothing is said, and the token is dropped. + said := len(r.said) + if err := r.h.Heard(stdio.OpExpire, NodeKey("unseen"), nil); err != nil || len(r.said) != said || !slices.Contains(r.vault.calls, "drop unseen") { + t.Errorf("an unseen expiry: %v, said %v, vault %v", err, r.events(), r.vault.calls) } } // 774/S58/AC2 and AC3: the drop is asked of the vault only from an ending event, acknowledged only once the vault answers, and // asked again when its first answer fails; and neither an event nor the bucket ever holds the token, sealed or not. -func TestTheDropFollowsOnlyAnEndAndNothingHoldsTheToken(t *testing.T) { +func theDropFollowsOnlyAnEndAndNothingHoldsTheToken(t *testing.T) { r := newRig(t) a, _ := r.h.Issue("person.operator", "ace", "a new machine", 0) _ = r.h.Warranted(yes, askWarrant(a.Ask)) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go new file mode 100644 index 0000000..bb157f8 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "os" + "testing" +) + +// The module's unit tests, as two tables (the repository's test budget): the operator's side (issue, the tokens verb, the +// manifest, the read of the bus node's loaded list) and the joining node's side (redeem, confirm and every end). + +// TestIssueTokensAndTheManifest is 774/S54 and S55, and the read of whether the bus node loaded a user. +func TestIssueTokensAndTheManifest(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"S55/AC1 issue refuses naming every finding", issueRefusesNamingEveryFinding}, + {"S55/AC2 two issues at once hold two addresses and a yes makes the token", twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken}, + {"S55/AC3 a declined or expired ask takes both keys; a vault failure keeps the enrolment for again", aDeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain}, + {"S54/AC1 tokens answers each stage and relays the sealed token of one node", tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode}, + {"S54/AC2 the manifest says what the code does", theManifestSaysWhatTheCodeDoes}, + {"S56 the bus node's loaded list is read from the grants part's pass", theLoadedListIsReadFromTheGrantsPass}, + } { + t.Run(c.name, c.run) + } +} + +// TestRedeemConfirmAndEveryEnd is 774/S56, S57 and S58. +func TestRedeemConfirmAndEveryEnd(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"S56/AC1 redeem refuses what it must and spends the token once", redeemRefusesWhatItMustAndSpendsTheTokenOnce}, + {"S56/AC2 a redeem stopped midway is finished by the same keys again", aRedeemStoppedMidwayIsFinishedByTheSameKeysAgain}, + {"S57/AC1 confirm is the node's own and ends the enrolment", confirmIsTheNodesOwnAndEndsTheEnrolment}, + {"S58/AC1-AC2 revoke ends at any stage and an expiry marker says expired", revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired}, + {"S58/AC3 the drop follows only an end and nothing holds the token", theDropFollowsOnlyAnEndAndNothingHoldsTheToken}, + } { + t.Run(c.name, c.run) + } +} + +// The key pass of mesh-grants.user-list holds the part's whole pass result; testdata/grants-pass.json was written by +// mesh-grants' own Result (json.Marshal of grants.Result with CredentialsOf), not written by hand: node.ace is in the shadow +// list and loaded, node.g14 in the shadow list and not loaded yet. +func theLoadedListIsReadFromTheGrantsPass(t *testing.T) { + raw, err := os.ReadFile("testdata/grants-pass.json") + if err != nil { + t.Fatal(err) + } + loaded := loadedVia(memRead{"mesh-grants.user-list": {"pass": string(raw)}}) + for user, want := range map[string]bool{"node.ace": true, "node.g14": false, "node.none": false} { + if got, err := loaded(user); err != nil || got != want { + t.Errorf("%s loaded = %v, %v; want %v", user, got, err, want) + } + } + if _, err := loadedVia(memRead{"mesh-grants.user-list": {}})("node.ace"); err == nil { + t.Error("no pass composed read as an answer") + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go index f6dbdde..8e7ef81 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -20,7 +20,8 @@ import ( type Vault interface { // Token asks the vault to make the token and its hash for node, expiring at expires, and answers what the joining // machine must trust (the bus's address and fingerprint, the vault's signing key). No token is in the answer. - Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) + // The token carries tunnel, the node's first tunnel built at issue (ADR 0587 §3, insight of 2026-11-02). + Token(node string, expires time.Time, tunnel delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) // Sealed asks the vault for node's token sealed to the operator key, as the vault sealed it. Sealed(node string) (string, error) // Drop asks the vault to drop node's token and its hash; it answers only once both are gone. @@ -159,6 +160,46 @@ func (h *Holder) placements() (map[string]delivery.Placement, error) { return p, nil } +// firstTunnel is the joining node's first tunnel as the token carries it (ADR 0587 §3, insight of 2026-11-02; as the +// controller's `token issue` builds it): the hub's tunnel key from its record (the key its tunnel holds; on an adopted hub +// that has not written one, the found tunnel's key kept in its placement), the hub's endpoint from its placement, and the +// node's address on the private network. A mesh with no hub, or a hub with no key or no endpoint, has no tunnel to join +// through, and that is said. +func (h *Holder) firstTunnel(placements map[string]delivery.Placement, address string) (delivery.TokenTunnel, error) { + var hub *delivery.Placement + for _, p := range placements { + if p.Decided.Hub { + p := p + hub = &p + } + } + if hub == nil { + return delivery.TokenTunnel{}, errors.New("placement names no hub, so there is no tunnel to join through") + } + key := "" + if k, err := delivery.HaveNodeKey(hub.Node); err == nil { + if raw, found, err := h.Read.Get("node-engine.have", k); err == nil && found { + if hn, err := delivery.DecodeHaveNode(raw); err == nil && hn.Record != nil { + key = hn.Record.TunnelKey + } + } + } + if key == "" && hub.Decided.Tunnel != nil { + key = hub.Decided.Tunnel.PublicKey + } + var missing []string + if key == "" { + missing = append(missing, "no tunnel key in its record") + } + if hub.Decided.Endpoint == "" { + missing = append(missing, "no endpoint in its placement") + } + if len(missing) > 0 { + return delivery.TokenTunnel{}, fmt.Errorf("the hub %s has %s, so there is no tunnel to join through", hub.Node, strings.Join(missing, " and ")) + } + return delivery.TokenTunnel{HubKey: key, HubEndpoint: hub.Decided.Endpoint, Address: address + "/32"}, nil +} + // hasRecord says the node already writes a record (a node of the mesh, ADR 0587 §4: "a node record ... holds it"). func (h *Holder) hasRecord(node string) bool { keys, err := h.Read.Keys("node-engine.have") diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go index 6bf05ad..7ae78d2 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -143,7 +143,7 @@ type standInVault struct { ctx context.Context logins jetstream.KeyValue mu sync.Mutex - tokens map[string]bool + tokens map[string]delivery.Token calls []string } @@ -153,9 +153,10 @@ func (v *standInVault) note(c string) { v.mu.Unlock() } -func (v *standInVault) Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) { +// Token makes the token as the vault does: the bus it states, its signing key, a secret, and the first tunnel it is handed. +func (v *standInVault) Token(node string, expires time.Time, tunnel delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { v.mu.Lock() - v.tokens[node] = true + v.tokens[node] = delivery.Token{Version: 1, Node: node, Broker: "bus:4222", Fingerprint: "f", Signer: make([]byte, 32), Secret: "secret-" + node, Tunnel: &tunnel} v.mu.Unlock() v.note("token " + node) return delivery.EnrolmentTokenAnswer{Node: node, Broker: "bus:4222", Fingerprint: "f", Signer: "s"}, nil @@ -219,13 +220,16 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { } enrolments := mk(delivery.EnrolmentsBucket, true) logins, lease, placement := mk(delivery.NodeLoginsBucket, false), mk("mesh-controller_lease", false), mk("mesh-delivery-assigner_placement", false) - mk("node-engine_have", false) + have := mk("node-engine_have", false) if _, err := lease.Put(ctx, "holder", []byte(`{"handed-to":"mesh-grants","handed-by":"operator"}`)); err != nil { t.Fatal(err) } if _, err := placement.Put(ctx, "hub.node", []byte(hubPlacement)); err != nil { t.Fatal(err) } + if _, err := have.Put(ctx, "hub.node", []byte(hubRecord)); err != nil { + t.Fatal(err) + } // The seats' events are kept on a stream of the events, as the bus keeps them, and heard on their real subjects. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_EVENTS", Subjects: []string{"mesh.seat.mesh-enrolment.event.>", "mesh.seat.mesh-placement.event.>"}}); err != nil { @@ -245,7 +249,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { events := func() []string { heardMu.Lock(); defer heardMu.Unlock(); return slices.Clone(heard) } store := &busStore{ctx: ctx, js: js, kv: enrolments} - vault := &standInVault{ctx: ctx, logins: logins, tokens: map[string]bool{}} + vault := &standInVault{ctx: ctx, logins: logins, tokens: map[string]delivery.Token{}} asker := &fakeAsker{} h := &Holder{Store: store, Read: busReader{ctx, js}, Vault: vault, Asker: asker, AskLasts: time.Hour, Say: func(event string, body any, id string) error { @@ -334,6 +338,18 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { t.Fatal(err) } waitFor("the user enrol.ace is composed", func() bool { return hasUser("enrol.ace") }) + // The token carries the first tunnel built at issue from the hub's placement and record (ADR 0587 §3, insight of + // 2026-11-02). Everything mesh-enrolment builds is in it; the joining machine's own tunnel key and the private network's + // range are not, because the seat's enrolment-token takes hub_key, hub_endpoint and address alone (reported in round 1). + vault.mu.Lock() + tok := vault.tokens["ace"] + vault.mu.Unlock() + if tok.Tunnel == nil || tok.Tunnel.HubKey != "hub-tunnel-key" || tok.Tunnel.HubEndpoint != "192.0.2.1:51820" || tok.Tunnel.Address != a.Address+"/32" { + t.Fatalf("the token's tunnel %+v", tok.Tunnel) + } + if missing := tok.Missing(); !slices.Equal(missing, []string{"the tunnel key it was issued for", "the private network's range"}) { + t.Errorf("the token misses %v", missing) + } args := redeemArgs("ace", 1, "serving-ace") if _, err := h.Redeem("enrol.somebody", args); err == nil { t.Fatal("redeemed by another caller") @@ -361,7 +377,8 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { waitFor("the vault is asked to drop the token after the end", func() bool { vault.mu.Lock() defer vault.mu.Unlock() - return slices.Contains(vault.calls, "drop ace") && !vault.tokens["ace"] + _, held := vault.tokens["ace"] + return slices.Contains(vault.calls, "drop ace") && !held }) if keys, _ := store.Keys(); len(keys) != 0 { t.Errorf("an ended enrolment leaves keys %v", keys) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index 86b414b..969a89d 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -20,8 +20,6 @@ type Event struct { Node string `json:"node"` Enrolment Enrolment `json:"enrolment"` Reason string `json:"reason,omitempty"` - // Issued names the enrolment-issued event that carried the content (for enrolment-expired, whose key is gone). - Issued string `json:"issued-event,omitempty"` } // eventID is the message id an event of e is said under: the bus keeps one message of it, so an event said again after a @@ -86,6 +84,11 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw findings = append(findings, "an enrolment of "+node+" is open already: revoke it first") } } + if perr == nil { + if _, err := h.firstTunnel(placements, "0.0.0.0"); err != nil { + findings = append(findings, err.Error()) + } + } if len(findings) > 0 { return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %s", node, strings.Join(findings, "; ")) } @@ -235,7 +238,15 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { return fmt.Errorf("the warrant %s is not recorded in the hand-act log, so no token is made (mesh-enrolment.again asks again): %w", w.Ask, err) } } - if _, err := h.Vault.Token(e.Node, e.Expires); err != nil { + placements, err := h.placements() + if err != nil { + return fmt.Errorf("placement cannot be read, so the token's tunnel is not built (mesh-enrolment.again asks again): %w", err) + } + tunnel, err := h.firstTunnel(placements, e.Address) + if err != nil { + return fmt.Errorf("the token's tunnel is not built (mesh-enrolment.again asks again): %w", err) + } + if _, err := h.Vault.Token(e.Node, e.Expires, tunnel); err != nil { return fmt.Errorf("the vault did not make the token of %s (mesh-enrolment.again asks the operator again): %w", e.Node, err) } e.Stage = StageLive diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go index f94331a..5fcaacb 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go @@ -8,13 +8,14 @@ import ( "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) var yes = asks.Option{ID: "yes"} // 774/S55/AC1: issue refuses, naming every finding, what it must: the lease with the controller, a bad name, a name a node // record, a placement key or an open enrolment holds, and an expiry past a day. -func TestIssueRefusesNamingEveryFinding(t *testing.T) { +func issueRefusesNamingEveryFinding(t *testing.T) { cases := []struct { name string setup func(r *rig) @@ -30,6 +31,14 @@ func TestIssueRefusesNamingEveryFinding(t *testing.T) { r.rd["node-engine.have"]["ace"] = `{}` }, "ace", 0, []string{"ace has a placement key already"}}, {"an open enrolment", func(r *rig) { r.seed("ace", "10.9.0.2", StageLive) }, "ace", 0, []string{"an enrolment of ace is open already"}}, + // The token carries the first tunnel (ADR 0587 §3, insight of 2026-11-02): no hub, or a hub with no key or endpoint. + {"no hub", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `"hub":true,`, "", 1) + }, "ace", 0, []string{"placement names no hub"}}, + {"a hub with no key and no endpoint", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `"endpoint":"192.0.2.1:51820",`, "", 1) + delete(r.rd["node-engine.have"], "hub.node") + }, "ace", 0, []string{"the hub hub has no tunnel key in its record and no endpoint in its placement"}}, } for _, c := range cases { r := newRig(t) @@ -51,7 +60,7 @@ func TestIssueRefusesNamingEveryFinding(t *testing.T) { // 774/S55/AC2: two issues at once for two nodes hold two addresses; a yes makes the token, moves both keys to live with the // token's expiry, records the warrant, says enrolment-issued, and a second answer changes nothing. -func TestTwoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { +func twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { r := newRig(t) var wg sync.WaitGroup addrs := make([]string, 2) @@ -86,6 +95,9 @@ func TestTwoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { len(r.warr) != 1 || r.warr[0] != e.Ask || r.events()[0] != EventIssued { t.Errorf("live %+v, ttl %v/%v, warrants %v, events %v", live, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey(live.Address)], r.warr, r.events()) } + if got, want := r.vault.tunnels["ace"], (delivery.TokenTunnel{HubKey: "hub-tunnel-key", HubEndpoint: "192.0.2.1:51820", Address: live.Address + "/32"}); got != want { + t.Errorf("the token's tunnel %+v, want %+v", got, want) + } if strings.Contains(string(r.said[0].Body), "secret") || r.said[0].ID != eventID(EventIssued, live) { t.Errorf("the event %s carries a secret-like word or no stable id", r.said[0].Body) } @@ -96,7 +108,7 @@ func TestTwoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { // 774/S55/AC3: a declined or expired ask, heard, takes both keys away, and enrolment-refused carries the key's content; a // vault that fails leaves the enrolment at asked, for `again` to ask once more. -func TestADeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain(t *testing.T) { +func aDeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain(t *testing.T) { r := newRig(t) a, _ := r.h.Issue("person.operator", "ace", "a new machine", 0) if err := r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: a.Ask}); err != nil { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 3ace80b..74be06e 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -114,8 +114,9 @@ func (r memRead) Get(state, key string) ([]byte, bool, error) { // fakeVault records its calls in order and fails on the verbs named in fail, once each. type fakeVault struct { - calls []string - fail map[string]bool + calls []string + fail map[string]bool + tunnels map[string]delivery.TokenTunnel } func (v *fakeVault) do(what string) error { @@ -127,7 +128,11 @@ func (v *fakeVault) do(what string) error { return nil } -func (v *fakeVault) Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) { +func (v *fakeVault) Token(node string, expires time.Time, tunnel delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { + if v.tunnels == nil { + v.tunnels = map[string]delivery.TokenTunnel{} + } + v.tunnels[node] = tunnel return delivery.EnrolmentTokenAnswer{Node: node, Broker: "b", Fingerprint: "f", Signer: "s"}, v.do("token " + node) } func (v *fakeVault) Sealed(node string) (string, error) { @@ -191,14 +196,17 @@ type rig struct { placeErr error } -const hubPlacement = `{"v":1,"node":"hub","decided":{"hub":true,"address":"10.9.0.1","tunnel":{"interface":"wg0","address":"10.9.0.1/24","range":"10.9.0.0/24"}},"modules":[]}` +// hubRecord is the hub's own record in node-engine.have: its tunnel key, which the token's tunnel names. +const hubRecord = `{"v":1,"record":{"v":1,"node":"hub","tunnel-key":"hub-tunnel-key"}}` + +const hubPlacement = `{"v":1,"node":"hub","decided":{"hub":true,"endpoint":"192.0.2.1:51820","address":"10.9.0.1","tunnel":{"interface":"wg0","address":"10.9.0.1/24","range":"10.9.0.0/24"}},"modules":[]}` func newRig(t *testing.T) *rig { r := &rig{t: t, st: newMem(), vault: &fakeVault{fail: map[string]bool{}}, asker: &fakeAsker{}, now: time.Date(2026, 11, 2, 10, 0, 0, 0, time.UTC), loaded: true} r.rd = memRead{ "mesh-controller.lease": {"holder": `{"handed-to":"mesh-grants","handed-by":"operator"}`}, "mesh-delivery-assigner.placement": {"hub.node": hubPlacement}, - "node-engine.have": {}, + "node-engine.have": {"hub.node": hubRecord}, } r.h = &Holder{Store: r.st, Read: r.rd, Vault: r.vault, Asker: r.asker, AskLasts: time.Hour, Say: func(event string, body any, id string) error { @@ -210,6 +218,7 @@ func newRig(t *testing.T) *rig { }, AddNode: func(a AddNode) error { r.added = append(r.added, a) + r.vault.calls = append(r.vault.calls, "place "+a.Change.Node) if r.placeErr == nil { // placement writes the node as the assigner does on the warrant n := a.Change.Node r.rd["mesh-delivery-assigner.placement"][n+".node"] = `{"v":1,"node":"` + n + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go index a9ae270..5016954 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -137,21 +137,25 @@ func loadedVia(rd Reader) func(string) (bool, error) { if !found { return false, errors.New("the grants part has composed no list yet") } - var st struct { - Compare struct { - Unread string `json:"unread"` - } `json:"compare"` - Credentials struct { - Users []struct { - User string `json:"user"` - InShadow bool `json:"in_shadow"` - Loaded bool `json:"loaded"` - } `json:"users"` - } `json:"credentials"` + // The key holds the part's whole pass result (mesh-grants Result): the list state sits under user_list. + var pass struct { + UserList struct { + Compare struct { + Unread string `json:"unread"` + } `json:"compare"` + Credentials struct { + Users []struct { + User string `json:"user"` + InShadow bool `json:"in_shadow"` + Loaded bool `json:"loaded"` + } `json:"users"` + } `json:"credentials"` + } `json:"user_list"` } - if err := json.Unmarshal(raw, &st); err != nil { + if err := json.Unmarshal(raw, &pass); err != nil { return false, fmt.Errorf("the grants part's list state does not read: %w", err) } + st := pass.UserList if st.Compare.Unread != "" { return false, errors.New("the bus node's loaded list cannot be read: " + st.Compare.Unread) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go index 38f38af..df8760b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -13,7 +13,8 @@ import ( // Redeem is the joining node-engine's one call (ADR 0587 §5, §6). It answers only a caller the runtime proves to be // enrol., for a key at stage live (or at redeeming with the same keys' digest), checks the proof against the identity key // presented, and then, each step read back and none repeated that is done: spends the token (the key to redeeming, by -// compare-and-set, with no expiry), asks the vault for the node's account and the certificate, writes the node into placement, +// compare-and-set, with no expiry), writes the node into placement (the vault reads the node's address from it), asks the vault +// for the node's account and the certificate of the serving key presented, // waits for the bus node to load node., and answers the whole, signed by the vault. Called again with the same keys it // finishes what is not done and answers the same. func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer, error) { @@ -37,6 +38,9 @@ func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer if err != nil { return delivery.RedeemAnswer{}, err } + if err := h.place(e, a); err != nil { + return delivery.RedeemAnswer{}, err + } account, err := h.Vault.Account(e.Node, a.SealingKey, digest) if err != nil { return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not give the account of %s (call redeem again with the same keys): %w", e.Node, err) @@ -45,9 +49,6 @@ func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer if err != nil { return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) } - if err := h.place(e, a); err != nil { - return delivery.RedeemAnswer{}, err - } if h.Loaded != nil { loaded, err := h.Loaded(NodeUser(e.Node)) if err != nil { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go index 9dad26f..81599ff 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/ed25519" "encoding/base64" + "slices" "testing" "time" @@ -22,7 +23,7 @@ func redeemArgs(node string, seed byte, serving string) delivery.Redeem { // 774/S56/AC1: a redeem from another caller, with a bad proof, after the expiry, or with other keys than the first, is // refused (one row each), and a good one spends the token, writes the node into placement on the issue's warrant and // answers once the bus node loaded the account. -func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { +func redeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { good := redeemArgs("ace", 1, "serving-a") bad := good raw, _ := good.ProofBytes() @@ -81,6 +82,10 @@ func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { r.added[0].Change.Keys.Digest() != good.KeysDigest() || r.added[0].Change.Decided.Address != "10.9.0.2" || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { t.Errorf("placement %+v, answer %+v", r.added, ans) } + // Placement is written before the vault is asked: the vault reads the node's address from it. + if place, account := slices.Index(r.vault.calls, "place ace"), slices.Index(r.vault.calls, "account ace"); place < 0 || account < place { + t.Errorf("the vault was asked before placement held the node: %v", r.vault.calls) + } r.loaded = false if _, err := r.h.Redeem("enrol.ace", good); err == nil { t.Error("answered before the bus node loaded the account") @@ -89,7 +94,7 @@ func TestRedeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { // 774/S56/AC2: a redeem stopped after the vault's call, or after the placement call, called again with the same keys, // finishes and answers the same, and the vault minted once (it is asked for the same account). -func TestARedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { +func aRedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { good := redeemArgs("ace", 1, "serving-a") whole := newRig(t) whole.seed("ace", "10.9.0.2", StageLive) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json b/modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json new file mode 100644 index 0000000..1b04918 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json @@ -0,0 +1,53 @@ +{ + "user_list": { + "v": 1, + "mode": "shadow", + "inputs": "", + "digest": "", + "users": 0, + "left_out": null, + "compare": { + "shadow": 0, + "loaded": 0, + "added": null, + "dropped": null, + "differ": null, + "same": 0 + }, + "credentials": { + "users": [ + { + "user": "node.ace", + "in_shadow": true, + "loaded": true, + "hash_held": true, + "generation": 1, + "would": "same" + }, + { + "user": "node.g14", + "in_shadow": true, + "loaded": false, + "hash_held": true, + "generation": 1, + "would": "adds" + } + ], + "hashes_held": 2, + "left_out": [], + "loses_login": [], + "adds": [ + "node.g14" + ] + }, + "gaps": null, + "at": "0001-01-01T00:00:00Z" + }, + "revision": 0, + "written": false, + "memberships": 0, + "memberships_written": 0, + "assertions": 0, + "assertions_written": 0, + "published_or_asserted": "" +} \ No newline at end of file diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index ab4b890..75782fd 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -24,7 +24,7 @@ func (r *rig) seed(node, address string, stage string) { // 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, the fill and whether issuing is on; // for one node at live it relays the vault's sealed token and keeps nothing; and the manifest says what the code does. -func TestTokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { +func tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { r := newRig(t) r.seed("ace", "10.9.0.2", StageAsked) r.seed("g14", "10.9.0.3", StageLive) @@ -65,7 +65,7 @@ func TestTokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { } } -func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { +func theManifestSaysWhatTheCodeDoes(t *testing.T) { raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) if err != nil { t.Fatal(err) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go index 7ff7953..8ef9ff9 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go @@ -39,8 +39,11 @@ func answer[T any](raw json.RawMessage, err error) (T, error) { return out, nil } -func (v vaultOver) Token(node string, expires time.Time) (delivery.EnrolmentTokenAnswer, error) { - return answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "expires": expires.UTC().Format(time.RFC3339)})) +func (v vaultOver) Token(node string, expires time.Time, t delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { + // The seat's tunnel takes these three fields and no other (catalogue's enrolment-token). + tunnel := map[string]any{"hub_key": t.HubKey, "hub_endpoint": t.HubEndpoint, "address": t.Address} + return answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, + "expires": expires.UTC().Format(time.RFC3339), "tunnel": tunnel})) } func (v vaultOver) Sealed(node string) (string, error) { From 40bbc7e0d3c1151603d92ad63d6b7d1d550bbb45 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:13:43 +0100 Subject: [PATCH 12/20] mesh-enrolment: take the tunnel key at issue, place the node on the yes, take it off again at an unconfirmed end The operator's decision of 2026-11-02 01:05, on mesh-sdk 4224665: - issue takes overlay_key, the machine's tunnel public key as `nox-mesh-host key` prints it, and refuses a missing one. The token's first tunnel is whole: key, address, range (the hub's tunnel range in placement, the one the controller's token carries today; a hub with none is a finding, never a guess), hub key and hub endpoint. - On the operator's yes the node is proposed into placement at once (add-node with its tunnel key alone, on the warrant), and only then is the token made, so the hub peers it before redeem. Redeem writes no placement: it refuses a tunnel key other than the one issued for, or a node placement does not hold yet, and the certificate's call carries all four keys presented. - A revoke, a refusal or an expiry before confirm takes the node off placement (remove-node, same warrant) before the end is said, and the end brings the token's drop. - The assigner applies a remove-node: the node's key deleted at once, with no ask, placement-changed carrying the warrant. - The tunnel key is kept as one more member of the node key's JSON, beside delivery.Enrolment, until the contract carries it. Tests: rows in the existing tables and both integration tests (the token's Missing() is empty; the node is placed before redeem and leaves placement when its enrolment expires); no function added, mesh-delivery stays at 816. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../integration_test.go | 22 +++- .../cmd/mesh-delivery-assigner/seat.go | 31 +++++ .../mesh-delivery-assigner/seat_tools_test.go | 24 +++- .../internal/assign/seat.go | 10 ++ .../mesh-enrolment/cmd/mesh-enrolment/end.go | 11 +- .../cmd/mesh-enrolment/end_test.go | 11 +- .../cmd/mesh-enrolment/enrolment.go | 40 ++++++- .../cmd/mesh-enrolment/holder.go | 106 ++++++++++++++---- .../cmd/mesh-enrolment/integration_test.go | 39 ++++--- .../cmd/mesh-enrolment/issue.go | 25 ++++- .../cmd/mesh-enrolment/issue_test.go | 38 +++++-- .../cmd/mesh-enrolment/kit_test.go | 18 ++- .../cmd/mesh-enrolment/redeem.go | 49 +++----- .../cmd/mesh-enrolment/redeem_test.go | 30 +++-- .../cmd/mesh-enrolment/tokens_test.go | 7 +- .../cmd/mesh-enrolment/tools.go | 18 +-- .../cmd/mesh-enrolment/vault.go | 9 +- 17 files changed, 369 insertions(+), 119 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index ec6fb24..1457c91 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1141,7 +1141,7 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { <-heard } joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`, - "changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"identity":"aWQ=","sealing":"c2Vh"}}]`}, "novox.mesh-enrolment") + "changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"overlay":"dHVubmVs"}}]`}, "novox.mesh-enrolment") if err != nil { t.Fatal(err) } @@ -1159,4 +1159,24 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(delivery.PlacementProposalsBucket, joined.(proposed).Proposal.ID) { t.Error("an ask was raised for the joining node, or its proposal stays open") } + // Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and + // placement-changed carries the warrant (the operator's decision of 2026-11-02 01:05). + for len(heard) > 0 { + <-heard + } + if _, err := verbs.doPropose(map[string]any{"why": "enrolment of ace revoked", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`, + "changes": `[{"op":"remove-node","node":"ace"}]`}, "novox.mesh-enrolment"); err != nil { + t.Fatal(err) + } + select { + case e := <-heard: + if e.Warrant == nil || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" { + t.Errorf("placement-changed for the removed node = %+v", e) + } + case <-time.After(15 * time.Second): + t.Fatal("no placement-changed heard for the removed node") + } + if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err == nil { + t.Error("the removed node's key stays") + } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index 3dc3358..aba87c5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -560,6 +560,10 @@ func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, r } laid, findings := delivery.LayChanges(current, prop.Changes) node := prop.Changes[0].Node + if prop.Changes[0].Op == delivery.OpRemoveNode { + p.removeNode(ctx, prop, current, findings) + return + } value, ok := laid[node] if len(findings) > 0 || !ok { p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; ")) @@ -582,6 +586,33 @@ func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, r []delivery.PlacementChange{c}) } +// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's +// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant. +// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal). +func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) { + node := prop.Changes[0].Node + before, ok := current[node] + if len(findings) > 0 || !ok { + p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; ")) + return + } + rev, err := p.writeNode(ctx, node, nil) + if err != nil { + p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error()) + return + } + c := delivery.ChangeOf(before, delivery.Placement{Node: node}) + c.Before, c.After = prop.Base[node], rev + p.deleteCandidates(ctx, prop.ID) + p.deletePulses(ctx, prop.ID) + _ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID) + p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil) + w := prop.Warrant + p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant, + Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)}, + []delivery.PlacementChange{c}) +} + // followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the // controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774). func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index f4f80f4..2636592 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -154,7 +154,7 @@ func TestTheSeatsVerbs(t *testing.T) { // the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused. addNode := func(by, warrant string) error { _, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace", - "decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"identity": "aWQ=", "sealing": "c2Vh"}}}}, by) + "decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by) return err } good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}` @@ -172,6 +172,28 @@ func TestTheSeatsVerbs(t *testing.T) { if err := addNode("novox.mesh-enrolment", good); err != nil { t.Errorf("an add-node from mesh-enrolment: %v", err) } + // A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from + // mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete; + // of a node running modules, or with another node's warrant, it is refused. + joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}}) + st[delivery.PlacementState].values["placement.joiner"] = joiner + removeNode := func(node, warrant string) (any, error) { + return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant, + "changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment") + } + for _, c := range []struct{ node, warrant, want string }{ + {"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"}, + {"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"}, + } { + if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want) + } + } + if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil { + t.Errorf("a remove-node from mesh-enrolment: %v", err) + } else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) { + t.Errorf("the remove-node proposed %+v", rm) + } if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") { t.Errorf("a hand-back: %v", err) } diff --git a/modules/mesh-delivery-assigner/internal/assign/seat.go b/modules/mesh-delivery-assigner/internal/assign/seat.go index 3b2dc1e..e9f9990 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat.go @@ -67,6 +67,16 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla out[node] = &pl p.Base[node] = in.Revisions[node] } + // A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its + // candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go. + for _, node := range p.RemovedNodes() { + if _, ok := in.Current[node]; !ok { + continue + } + delete(all, node) + out[node] = nil + p.Base[node] = in.Revisions[node] + } if len(findings) == 0 && len(out) == 0 { findings = append(findings, "the changes leave every placement as it is") } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go index 545f788..7d2584e 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -10,8 +10,8 @@ import ( ) // Revoke ends an enrolment at any stage (ADR 0587 §5), by a person's call: at asked it withdraws the ask; at redeeming it also -// undoes what redeem had not finished, the vault retiring a node account it already minted (a node already written into -// placement stays there and is removed, if wanted, as any node is removed). It only takes access away, so it needs no Approve; +// undoes what redeem had not finished, the vault retiring a node account it already minted, and a node the yes added to +// placement is taken off it again. It only takes access away, so it needs no Approve; // it is recorded in the hand-act log. enrolment-revoked carries the key's content and the why; the token is then dropped by // the durable consumer of that event, never before. A revoke that stopped is finished by calling it again. func (h *Holder) Revoke(caller, node, why string) (map[string]any, error) { @@ -42,6 +42,10 @@ func (h *Holder) Revoke(caller, node, why string) (map[string]any, error) { return nil, fmt.Errorf("the vault did not retire the account minted for %s, so nothing is ended (call revoke again): %w", node, err) } } + // A node that entered placement on the yes and never confirmed leaves it again (the operator's decision of 2026-11-02 01:05). + if err := h.unplace(e, "revoked: "+why); err != nil { + return nil, fmt.Errorf("%s is not taken off placement, so nothing is ended (call revoke again): %w", node, err) + } out := map[string]any{"node": node, "was": e.Stage} if h.Act != nil { if err := h.Act("mesh-enrolment revoke of "+node+" at "+string(e.Stage), why+" (by "+caller+")", "enrolment-revoke"); err != nil { @@ -92,6 +96,9 @@ func (h *Holder) Heard(op, key string, value []byte) error { } return nil } + if err := h.unplace(e, "expired"); err != nil { + return fmt.Errorf("%s, whose enrolment expired, is not taken off placement (offered again): %w", node, err) + } body, err := delivery.EncodeEnrolmentExpiry(delivery.EnrolmentExpiry{Node: node, IssuedID: eventID(EventIssued, e)}) if err != nil { return err diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go index f913ed3..e812414 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "git.novox.be/novox/mesh-sdk/go/delivery" + "reflect" "slices" "strings" "testing" @@ -23,6 +24,11 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err != nil { t.Fatal(err) } + // The node the yes placed, never confirmed, leaves placement on the enrolment's warrant before the end is said. + if _, placed := r.rd["mesh-delivery-assigner.placement"]["ace.node"]; placed || len(r.added) == 0 || + !reflect.DeepEqual(r.added[len(r.added)-1].Change, delivery.PlacementProposalChange{Op: delivery.OpRemoveNode, Node: "ace"}) || r.added[len(r.added)-1].Warrant.Ask != "ask-ace" { + t.Errorf("revoked and still placed: %+v", r.added) + } if len(r.keys()) != 0 || !slices.Equal(r.events(), []string{EventRevoked}) || len(r.acts) != 1 || !slices.Contains(r.vault.calls, "retire ace") { t.Errorf("keys %v events %v acts %v vault %v", r.keys(), r.events(), r.acts, r.vault.calls) } @@ -42,6 +48,9 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { if err := r.h.Heard(stdio.OpExpire, NodeKey("novox"), nil); err != nil { t.Fatal(err) } + if _, placed := r.rd["mesh-delivery-assigner.placement"]["novox.node"]; placed { + t.Error("expired and still placed") + } last := r.said[len(r.said)-1] x, err := delivery.DecodeEnrolmentExpiry(last.Body) if err != nil || last.Event != EventExpired || x.Node != "novox" || !strings.HasPrefix(x.IssuedID, "enrolment-issued.novox.") { @@ -58,7 +67,7 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { // asked again when its first answer fails; and neither an event nor the bucket ever holds the token, sealed or not. func theDropFollowsOnlyAnEndAndNothingHoldsTheToken(t *testing.T) { r := newRig(t) - a, _ := r.h.Issue("person.operator", "ace", "a new machine", 0) + a, _ := r.h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0) _ = r.h.Warranted(yes, askWarrant(a.Ask)) if _, err := r.h.Tokens("ace", false); err != nil { // relays the sealed token t.Fatal(err) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go index f492b5d..2b7a106 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go @@ -50,11 +50,18 @@ const ( StageRedeeming = delivery.EnrolmentRedeeming ) -// Enrolment and Reservation are the seat's state contract (delivery.Enrolment, delivery.EnrolmentAddress). -type ( - Enrolment = delivery.Enrolment - Reservation = delivery.EnrolmentAddress -) +// Reservation is the seat's state contract for an address key (delivery.EnrolmentAddress). +type Reservation = delivery.EnrolmentAddress + +// Enrolment is a node key: the seat's state contract (delivery.Enrolment) and the machine's tunnel key given at issue, which +// the node enters placement with on the operator's yes and which redeem must present again (the operator's decision of +// 2026-11-02 01:05). The contract does not carry the tunnel key yet: it is kept as one more member of the same JSON object, +// which every reader of the contract decodes as before (delivery's decode ignores a member it does not know). Once +// delivery.Enrolment carries `overlay-key` this wrapper goes. +type Enrolment struct { + delivery.Enrolment + OverlayKey string `json:"overlay-key,omitempty"` +} const ( nodePrefix = "node." @@ -78,12 +85,33 @@ func CheckNode(node string) error { return nil } -func decodeEnrolment(raw []byte) (Enrolment, error) { return delivery.DecodeEnrolment(raw) } +func decodeEnrolment(raw []byte) (Enrolment, error) { + e, err := delivery.DecodeEnrolment(raw) + if err != nil { + return Enrolment{}, err + } + var tunnel struct { + OverlayKey string `json:"overlay-key"` + } + _ = json.Unmarshal(raw, &tunnel) + return Enrolment{Enrolment: e, OverlayKey: tunnel.OverlayKey}, nil +} // Encode is a value of the bucket as the seat's contract writes it: an Enrolment or a Reservation checked and versioned. func Encode(v any) (json.RawMessage, error) { switch x := v.(type) { case Enrolment: + raw, err := delivery.EncodeEnrolment(x.Enrolment) + if err != nil || x.OverlayKey == "" { + return raw, err + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, err + } + obj["overlay-key"], _ = json.Marshal(x.OverlayKey) + return json.Marshal(obj) + case delivery.Enrolment: return delivery.EncodeEnrolment(x) case Reservation: return delivery.EncodeEnrolmentAddress(x.Node) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go index 8e7ef81..a6ec725 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -1,6 +1,7 @@ package main import ( + "encoding/base64" "encoding/json" "errors" "fmt" @@ -30,8 +31,9 @@ type Vault interface { Account(node, sealingKey, keysDigest string) (delivery.NodeAccountAnswer, error) // Retire asks the vault to retire the account it minted for node. Retire(node string) error - // SignCertificate and SignReply sign through the vault's `sign`. - SignCertificate(node, servingKey string) (delivery.SignAnswer, error) + // SignCertificate and SignReply sign through the vault's `sign`; the certificate's call carries every public key the node + // presented, which the vault checks against the digest node-account was minted for. + SignCertificate(node string, keys delivery.JoiningKeys) (delivery.SignAnswer, error) SignReply(node string, reply []byte) (delivery.SignAnswer, error) // Signer is the vault's public signing key in base64, which the redeem answer names for the node to check it against its token. Signer() (string, error) @@ -51,7 +53,7 @@ type Holder struct { Asker Asker // Say publishes a seat event under a message id the bus keeps one of. Say func(event string, body any, id string) error - // AddNode asks mesh-placement to add the node (ADR 0587 §7). + // AddNode asks mesh-placement to add the node, or to take a node that never confirmed off again (ADR 0587 §7). AddNode func(AddNode) error // Warrant and Act record in the hand-act log, through the seat mesh-hand-acts: the operator's warrant for an ask, and an // act done by hand (a revoke). @@ -103,6 +105,59 @@ type AddNode struct { Warrant delivery.AddNodeWarrant } +// place adds the node into placement, with the address the operator approved and its tunnel key, on the issue's warrant +// (ADR 0587 §7; the operator's decision of 2026-11-02 01:05), unless placement holds it at that address already. +func (h *Holder) place(e Enrolment, placements map[string]delivery.Placement) error { + if p, ok := placements[e.Node]; ok && p.Decided.Address == e.Address { + return nil + } + w, err := h.warrantOf(e) + if err != nil { + return err + } + return h.AddNode(AddNode{Why: "enrolment of " + e.Node + ": " + e.Why, Warrant: w, + Change: delivery.PlacementProposalChange{Op: delivery.OpAddNode, Node: e.Node, Decided: &delivery.NodeDecided{Address: e.Address}, + Keys: &delivery.JoiningKeys{Overlay: e.OverlayKey}}}) +} + +// unplace takes a node that never confirmed off placement again, on the same warrant, when its enrolment ends revoked, refused +// or expired: the hub stops peering a machine that will not join. A node placement does not hold is left as it is. +func (h *Holder) unplace(e Enrolment, why string) error { + placements, err := h.placements() + if err != nil { + return fmt.Errorf("placement cannot be read: %w", err) + } + if _, ok := placements[e.Node]; !ok { + return nil + } + w, err := h.warrantOf(e) + if err != nil { + return err + } + return h.AddNode(AddNode{Why: "enrolment of " + e.Node + " ended unconfirmed: " + why, Warrant: w, + Change: delivery.PlacementProposalChange{Op: delivery.OpRemoveNode, Node: e.Node}}) +} + +func (h *Holder) warrantOf(e Enrolment) (delivery.AddNodeWarrant, error) { + if h.AddNode == nil || h.WarrantOf == nil { + return delivery.AddNodeWarrant{}, errors.New("no route to mesh-placement") + } + warrant, err := h.WarrantOf(e.Ask) + if err != nil { + return delivery.AddNodeWarrant{}, fmt.Errorf("the operator's warrant for %s cannot be read from the router: %w", e.Ask, err) + } + return delivery.AddNodeWarrant{Node: e.Node, Address: e.Address, Ask: e.Ask, Warrant: warrant}, nil +} + +// checkTunnelKey says a tunnel key is a WireGuard public key, as `nox-mesh-host key` prints it: 32 bytes in standard base64. +func checkTunnelKey(k string) error { + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(k)) + if err != nil || len(raw) != 32 { + return errors.New("no tunnel key is given (overlay_key: the line `nox-mesh-host key` prints on the new machine)") + } + return nil +} + // issuing says whether issue is on: only while the controller's lease is handed to a holder that is no controller (ADR 0587 // §4), and why either way. func (h *Holder) issuing() (bool, string) { @@ -161,11 +216,12 @@ func (h *Holder) placements() (map[string]delivery.Placement, error) { } // firstTunnel is the joining node's first tunnel as the token carries it (ADR 0587 §3, insight of 2026-11-02; as the -// controller's `token issue` builds it): the hub's tunnel key from its record (the key its tunnel holds; on an adopted hub +// controller's `token issue` builds it): the machine's own tunnel key given at issue, its address, the private network's +// range (privateRange), the hub's tunnel key from its record (the key its tunnel holds; on an adopted hub // that has not written one, the found tunnel's key kept in its placement), the hub's endpoint from its placement, and the // node's address on the private network. A mesh with no hub, or a hub with no key or no endpoint, has no tunnel to join // through, and that is said. -func (h *Holder) firstTunnel(placements map[string]delivery.Placement, address string) (delivery.TokenTunnel, error) { +func (h *Holder) firstTunnel(placements map[string]delivery.Placement, address, overlayKey string) (delivery.TokenTunnel, error) { var hub *delivery.Placement for _, p := range placements { if p.Decided.Hub { @@ -194,10 +250,14 @@ func (h *Holder) firstTunnel(placements map[string]delivery.Placement, address s if hub.Decided.Endpoint == "" { missing = append(missing, "no endpoint in its placement") } + prefix, rerr := privateRange(placements) if len(missing) > 0 { return delivery.TokenTunnel{}, fmt.Errorf("the hub %s has %s, so there is no tunnel to join through", hub.Node, strings.Join(missing, " and ")) } - return delivery.TokenTunnel{HubKey: key, HubEndpoint: hub.Decided.Endpoint, Address: address + "/32"}, nil + if rerr != nil { + return delivery.TokenTunnel{}, rerr + } + return delivery.TokenTunnel{Key: overlayKey, Address: address + "/32", Range: prefix.String(), HubKey: key, HubEndpoint: hub.Decided.Endpoint}, nil } // hasRecord says the node already writes a record (a node of the mesh, ADR 0587 §4: "a node record ... holds it"). @@ -214,10 +274,25 @@ func (h *Holder) hasRecord(node string) bool { return false } +// privateRange is the private network's range: the hub's tunnel range in its placement, as the controller's `token issue` +// writes it into the token today (the adopted tunnel's range). A mesh whose hub's placement names none has no range a token +// can carry, and that is said rather than guessed. +func privateRange(placements map[string]delivery.Placement) (netip.Prefix, error) { + for _, p := range placements { + if t := p.Decided.Tunnel; p.Decided.Hub && t != nil && t.Range != "" { + r, err := netip.ParsePrefix(t.Range) + if err != nil { + return netip.Prefix{}, fmt.Errorf("the hub %s's tunnel range %q is no range: %w", p.Node, t.Range, err) + } + return r.Masked(), nil + } + } + return netip.Prefix{}, errors.New("no private network is known: the hub's placement names no tunnel range") +} + // nextAddress is the lowest address of the private network that no placement key and no reservation holds (ADR 0587 §4), // after skip. The network is the hub's range; the hub's own address and the tunnel's are never given. func nextAddress(placements map[string]delivery.Placement, reserved map[string]bool, skip map[string]bool) (string, error) { - var prefix netip.Prefix taken := map[string]bool{} for _, p := range placements { if a := p.Decided.Address; a != "" { @@ -231,22 +306,11 @@ func nextAddress(placements map[string]delivery.Placement, reserved map[string]b taken[t.Address] = true } } - if t.Range != "" && p.Decided.Hub && !prefix.IsValid() { - if r, err := netip.ParsePrefix(t.Range); err == nil { - prefix = r.Masked() - } - } } } - if !prefix.IsValid() { - for _, p := range placements { - if a, err := netip.ParseAddr(p.Decided.Address); err == nil && p.Decided.Hub && a.Is4() { - prefix = netip.PrefixFrom(a, 24).Masked() - } - } - } - if !prefix.IsValid() { - return "", errors.New("no private network is known: the hub's placement names no range or address") + prefix, err := privateRange(placements) + if err != nil { + return "", err } for a := prefix.Addr().Next(); prefix.Contains(a); a = a.Next() { s := a.String() diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go index 7ae78d2..ecacd45 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -180,7 +180,7 @@ func (v *standInVault) Account(node, sealing, digest string) (delivery.NodeAccou return delivery.NodeAccountAnswer{Node: node, Login: delivery.FirstLogin(node), Account: "account-" + node}, nil } func (v *standInVault) Retire(node string) error { return nil } -func (v *standInVault) SignCertificate(node, serving string) (delivery.SignAnswer, error) { +func (v *standInVault) SignCertificate(node string, _ delivery.JoiningKeys) (delivery.SignAnswer, error) { return delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Certificate: "PEM"}, nil } func (v *standInVault) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { @@ -257,10 +257,17 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { _, err := js.Publish(ctx, "mesh.seat.mesh-enrolment.event."+event, raw, jetstream.WithMsgID(id)) return err }, + // mesh-placement, stood in: an add-node writes the node's key, a remove-node deletes it, each saying placement-changed. AddNode: func(a AddNode) error { - value := `{"v":1,"node":"` + a.Change.Node + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` - if _, err := placement.Put(ctx, a.Change.Node+".node", []byte(value)); err != nil { - return err + if a.Change.Op == delivery.OpRemoveNode { + if err := placement.Delete(ctx, a.Change.Node+".node"); err != nil { + return err + } + } else { + value := `{"v":1,"node":"` + a.Change.Node + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` + if _, err := placement.Put(ctx, a.Change.Node+".node", []byte(value)); err != nil { + return err + } } _, err := js.Publish(ctx, "mesh.seat.mesh-placement.event.placement-changed", []byte(`{}`)) return err @@ -330,7 +337,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { } // Issue, the operator's yes, redeem as enrol.ace, confirm as node.ace. - a, err := h.Issue("person.operator", "ace", "a new machine", 0) + a, err := h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0) if err != nil { t.Fatal(err) } @@ -338,19 +345,19 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { t.Fatal(err) } waitFor("the user enrol.ace is composed", func() bool { return hasUser("enrol.ace") }) - // The token carries the first tunnel built at issue from the hub's placement and record (ADR 0587 §3, insight of - // 2026-11-02). Everything mesh-enrolment builds is in it; the joining machine's own tunnel key and the private network's - // range are not, because the seat's enrolment-token takes hub_key, hub_endpoint and address alone (reported in round 1). + // The token is whole (ADR 0587 §3, insight of 2026-11-02): it carries the first tunnel built at issue, the machine's tunnel + // key given then, the private network's range and the hub's key and endpoint. vault.mu.Lock() tok := vault.tokens["ace"] vault.mu.Unlock() - if tok.Tunnel == nil || tok.Tunnel.HubKey != "hub-tunnel-key" || tok.Tunnel.HubEndpoint != "192.0.2.1:51820" || tok.Tunnel.Address != a.Address+"/32" { - t.Fatalf("the token's tunnel %+v", tok.Tunnel) + if missing := tok.Missing(); len(missing) != 0 || tok.Tunnel == nil || tok.Tunnel.Key != tunnelKey || tok.Tunnel.Address != a.Address+"/32" { + t.Fatalf("the token misses %v: tunnel %+v", missing, tok.Tunnel) } - if missing := tok.Missing(); !slices.Equal(missing, []string{"the tunnel key it was issued for", "the private network's range"}) { - t.Errorf("the token misses %v", missing) + // The node is in placement before redeem: the operator's yes placed it, so the hub peers it before the machine dials. + if ace, err := placement.Get(ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), a.Address) { + t.Fatalf("ace is not in placement before redeem: %v", err) } - args := redeemArgs("ace", 1, "serving-ace") + args := redeemArgsWith("ace", 1, "serving-ace", tunnelKey) if _, err := h.Redeem("enrol.somebody", args); err == nil { t.Fatal("redeemed by another caller") } @@ -385,7 +392,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { } // A token never redeemed expires on the bus: the holder hears the marker, says enrolment-expired and the token is dropped. - if _, err := h.Issue("person.operator", "g14", "a second machine", 2*time.Second); err != nil { + if _, err := h.Issue("person.operator", "g14", "a second machine", tunnelKey, 2*time.Second); err != nil { t.Fatal(err) } if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: asker.asked[len(asker.asked)-1]}); err != nil { @@ -400,4 +407,8 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { if hasUser("enrol.g14") { t.Error("the user enrol.g14 stays after its key expired") } + // It never confirmed, so it left placement again before its end was said. + if _, err := placement.Get(ctx, "g14.node"); err == nil { + t.Error("g14 stays in placement after its enrolment expired") + } } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index 969a89d..bfb5d9b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -46,7 +46,7 @@ type IssueAnswer struct { // address and reserves it by creating its key (the bus refuses a key that exists, so two issues never get one address); // writes the key at stage asked; and asks the operator, on a channel that proves who answered. The token is made only on // the operator's yes (Warranted). -func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnswer, error) { +func (h *Holder) Issue(caller, node, why, overlayKey string, lasts time.Duration) (IssueAnswer, error) { var findings []string if caller == "" || caller == stdio.Unproven { findings = append(findings, "the caller is not proven by the runtime, and an enrolment records who asked") @@ -57,6 +57,11 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw if strings.TrimSpace(why) == "" { findings = append(findings, "no why is given: the operator is asked why this machine joins") } + // The machine's tunnel key, made there by `nox-mesh-host key`: on the operator's yes the node enters placement with it, so + // the hub peers it before the machine redeems (the operator's decision of 2026-11-02 01:05). + if err := checkTunnelKey(overlayKey); err != nil { + findings = append(findings, err.Error()) + } if lasts == 0 { lasts = delivery.DefaultTokenLife } @@ -85,7 +90,7 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw } } if perr == nil { - if _, err := h.firstTunnel(placements, "0.0.0.0"); err != nil { + if _, err := h.firstTunnel(placements, "0.0.0.0", overlayKey); err != nil { findings = append(findings, err.Error()) } } @@ -93,7 +98,8 @@ func (h *Holder) Issue(caller, node, why string, lasts time.Duration) (IssueAnsw return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %s", node, strings.Join(findings, "; ")) } now := h.now() - e := Enrolment{Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, Issued: now, Expires: now.Add(lasts)} + e := Enrolment{Enrolment: delivery.Enrolment{Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, + Issued: now, Expires: now.Add(lasts)}, OverlayKey: overlayKey} e.Ask = "enrol-" + node + "-" + now.UTC().Format("20060102t150405") skip := map[string]bool{} for { @@ -214,7 +220,7 @@ func (h *Holder) byAsk(ask string) (Enrolment, Row, bool, error) { } // Warranted is the operator's answer, handed once by the asker (asker.Client.Listen): a no ends the enrolment; a yes records -// the warrant, asks the vault for the token, moves the key to live with the bus's expiry at the token's, and says +// the warrant, adds the node to placement with its tunnel key, asks the vault for the token, moves the key to live with the bus's expiry at the token's, and says // enrolment-issued (ADR 0587 §4, §5). func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { e, _, found, err := h.byAsk(w.Ask) @@ -242,10 +248,15 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { if err != nil { return fmt.Errorf("placement cannot be read, so the token's tunnel is not built (mesh-enrolment.again asks again): %w", err) } - tunnel, err := h.firstTunnel(placements, e.Address) + tunnel, err := h.firstTunnel(placements, e.Address, e.OverlayKey) if err != nil { return fmt.Errorf("the token's tunnel is not built (mesh-enrolment.again asks again): %w", err) } + // The node enters placement now, with its tunnel key, on this warrant: the hub peers it before the machine redeems over + // the tunnel (the operator's decision of 2026-11-02 01:05). Only then is the token made. + if err := h.place(e, placements); err != nil { + return fmt.Errorf("the node %s is not added to placement, so no token is made (mesh-enrolment.again asks again): %w", e.Node, err) + } if _, err := h.Vault.Token(e.Node, e.Expires, tunnel); err != nil { return fmt.Errorf("the vault did not make the token of %s (mesh-enrolment.again asks the operator again): %w", e.Node, err) } @@ -262,6 +273,10 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { // refuse ends an enrolment at stage asked: both keys go and enrolment-refused carries the key's content (ADR 0587 §4). func (h *Holder) refuse(e Enrolment, reason string) error { + // A yes whose token failed added the node already; a refusal takes it off again. + if err := h.unplace(e, reason); err != nil { + return fmt.Errorf("%s is not taken off placement, so its enrolment stays: %w", e.Node, err) + } if err := h.sayEvent(EventRefused, e, reason); err != nil { return fmt.Errorf("enrolment-refused of %s cannot be said, so its keys stay: %w", e.Node, err) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go index 5fcaacb..f622fb7 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go @@ -1,6 +1,7 @@ package main import ( + "slices" "strings" "sync" "testing" @@ -31,6 +32,9 @@ func issueRefusesNamingEveryFinding(t *testing.T) { r.rd["node-engine.have"]["ace"] = `{}` }, "ace", 0, []string{"ace has a placement key already"}}, {"an open enrolment", func(r *rig) { r.seed("ace", "10.9.0.2", StageLive) }, "ace", 0, []string{"an enrolment of ace is open already"}}, + {"no tunnel key and no range", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `,"range":"10.9.0.0/24"`, "", 1) + }, "ace", -1, []string{"no tunnel key is given", "names no tunnel range"}}, // The token carries the first tunnel (ADR 0587 §3, insight of 2026-11-02): no hub, or a hub with no key or endpoint. {"no hub", func(r *rig) { r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `"hub":true,`, "", 1) @@ -45,7 +49,11 @@ func issueRefusesNamingEveryFinding(t *testing.T) { if c.setup != nil { c.setup(r) } - _, err := r.h.Issue("person.operator", c.node, "a new machine", c.lasts) + key, lasts := tunnelKey, c.lasts + if lasts == -1 { + key, lasts = "", 0 + } + _, err := r.h.Issue("person.operator", c.node, "a new machine", key, lasts) if err == nil { t.Fatalf("%s: issued", c.name) } @@ -68,7 +76,7 @@ func twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - a, err := r.h.Issue("person.operator", n, "a new machine", 0) + a, err := r.h.Issue("person.operator", n, "a new machine", tunnelKey, 0) if err != nil { t.Error(err) } @@ -95,9 +103,19 @@ func twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { len(r.warr) != 1 || r.warr[0] != e.Ask || r.events()[0] != EventIssued { t.Errorf("live %+v, ttl %v/%v, warrants %v, events %v", live, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey(live.Address)], r.warr, r.events()) } - if got, want := r.vault.tunnels["ace"], (delivery.TokenTunnel{HubKey: "hub-tunnel-key", HubEndpoint: "192.0.2.1:51820", Address: live.Address + "/32"}); got != want { + if got, want := r.vault.tunnels["ace"], (delivery.TokenTunnel{Key: tunnelKey, Address: live.Address + "/32", Range: "10.9.0.0/24", + HubKey: "hub-tunnel-key", HubEndpoint: "192.0.2.1:51820"}); got != want { t.Errorf("the token's tunnel %+v, want %+v", got, want) } + // The yes added the node to placement with its tunnel key alone, on the warrant, before the token was made. + if p, tok := slices.Index(r.vault.calls, "add-node ace"), slices.Index(r.vault.calls, "token ace"); p < 0 || tok < p { + t.Errorf("the token was made before placement held the node: %v", r.vault.calls) + } + for _, a := range r.added { + if a.Change.Node == "ace" && (*a.Change.Keys != delivery.JoiningKeys{Overlay: tunnelKey} || a.Warrant.Ask != e.Ask || a.Change.Decided.Address != live.Address) { + t.Errorf("the add-node at the yes %+v", a) + } + } if strings.Contains(string(r.said[0].Body), "secret") || r.said[0].ID != eventID(EventIssued, live) { t.Errorf("the event %s carries a secret-like word or no stable id", r.said[0].Body) } @@ -110,20 +128,20 @@ func twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { // vault that fails leaves the enrolment at asked, for `again` to ask once more. func aDeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain(t *testing.T) { r := newRig(t) - a, _ := r.h.Issue("person.operator", "ace", "a new machine", 0) + a, _ := r.h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0) if err := r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: a.Ask}); err != nil { t.Fatal(err) } - if len(r.keys()) != 0 || r.events()[0] != EventRefused { - t.Fatalf("keys %v events %v", r.keys(), r.events()) + if len(r.keys()) != 0 || r.events()[0] != EventRefused || len(r.added) != 0 { + t.Fatalf("keys %v events %v placement %v", r.keys(), r.events(), r.added) } mustContain(t, string(r.said[0].Body), `"address":"`+a.Address+`"`) - b, _ := r.h.Issue("person.operator", "g14", "a new machine", 0) + b, _ := r.h.Issue("person.operator", "g14", "a new machine", tunnelKey, 0) r.h.Said(asker.Recorded, asks.Warrant{Ask: b.Ask, Outcome: asks.OutcomeExpired}, nil) if len(r.keys()) != 0 || r.events()[1] != EventRefused { t.Fatalf("keys %v events %v", r.keys(), r.events()) } - c, _ := r.h.Issue("person.operator", "novox", "a new machine", 0) + c, _ := r.h.Issue("person.operator", "novox", "a new machine", tunnelKey, 0) r.vault.fail["token novox"] = true if err := r.h.Warranted(yes, asks.Warrant{Ask: c.Ask}); err == nil { t.Fatal("a vault that failed was taken for a token") @@ -131,6 +149,10 @@ func aDeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain if e, _ := r.enrolment("novox"); e.Stage != StageAsked { t.Fatalf("stage %s", e.Stage) } + // The yes placed the node before the vault failed: a refusal now would take it off again (decline, below for ace). + if _, placed := r.rd["mesh-delivery-assigner.placement"]["novox.node"]; !placed { + t.Fatal("the yes did not place novox before the vault was asked") + } r.now = r.now.Add(time.Minute) again, err := r.h.Again("novox") if err != nil || again.Ask == c.Ask || len(r.asker.asked) != 4 { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 74be06e..45a17b6 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -117,6 +117,7 @@ type fakeVault struct { calls []string fail map[string]bool tunnels map[string]delivery.TokenTunnel + cert delivery.JoiningKeys // the keys the certificate's call carried } func (v *fakeVault) do(what string) error { @@ -143,7 +144,9 @@ func (v *fakeVault) Account(node, sealing, digest string) (delivery.NodeAccountA return delivery.NodeAccountAnswer{Node: node, Login: "node." + node, Account: "account-" + node}, v.do("account " + node) } func (v *fakeVault) Retire(node string) error { return v.do("retire " + node) } -func (v *fakeVault) SignCertificate(node, serving string) (delivery.SignAnswer, error) { +func (v *fakeVault) SignCertificate(node string, k delivery.JoiningKeys) (delivery.SignAnswer, error) { + serving := k.Serving + v.cert = k return delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Certificate: "PEM-" + serving}, v.do("certificate " + node) } func (v *fakeVault) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { @@ -199,6 +202,9 @@ type rig struct { // hubRecord is the hub's own record in node-engine.have: its tunnel key, which the token's tunnel names. const hubRecord = `{"v":1,"record":{"v":1,"node":"hub","tunnel-key":"hub-tunnel-key"}}` +// tunnelKey is a machine's tunnel public key as `nox-mesh-host key` prints it. +const tunnelKey = "dHVubmVsLWtleS1vZi10aGUtbmV3LW1hY2hpbmUtMDE=" + const hubPlacement = `{"v":1,"node":"hub","decided":{"hub":true,"endpoint":"192.0.2.1:51820","address":"10.9.0.1","tunnel":{"interface":"wg0","address":"10.9.0.1/24","range":"10.9.0.0/24"}},"modules":[]}` func newRig(t *testing.T) *rig { @@ -218,10 +224,14 @@ func newRig(t *testing.T) *rig { }, AddNode: func(a AddNode) error { r.added = append(r.added, a) - r.vault.calls = append(r.vault.calls, "place "+a.Change.Node) - if r.placeErr == nil { // placement writes the node as the assigner does on the warrant + r.vault.calls = append(r.vault.calls, a.Change.Op+" "+a.Change.Node) + if r.placeErr == nil { // placement writes or deletes the node as the assigner does on the warrant n := a.Change.Node - r.rd["mesh-delivery-assigner.placement"][n+".node"] = `{"v":1,"node":"` + n + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` + if a.Change.Op == delivery.OpRemoveNode { + delete(r.rd["mesh-delivery-assigner.placement"], n+".node") + } else { + r.rd["mesh-delivery-assigner.placement"][n+".node"] = `{"v":1,"node":"` + n + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` + } } return r.placeErr }, diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go index df8760b..93a8355 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -13,8 +13,9 @@ import ( // Redeem is the joining node-engine's one call (ADR 0587 §5, §6). It answers only a caller the runtime proves to be // enrol., for a key at stage live (or at redeeming with the same keys' digest), checks the proof against the identity key // presented, and then, each step read back and none repeated that is done: spends the token (the key to redeeming, by -// compare-and-set, with no expiry), writes the node into placement (the vault reads the node's address from it), asks the vault -// for the node's account and the certificate of the serving key presented, +// compare-and-set, with no expiry) once placement holds the node (written on the operator's yes; the vault reads the node's +// address there) and the tunnel key presented is the one issued for, asks the vault for the node's account and the certificate +// of the keys presented, // waits for the bus node to load node., and answers the whole, signed by the vault. Called again with the same keys it // finishes what is not done and answers the same. func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer, error) { @@ -34,18 +35,15 @@ func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer return delivery.RedeemAnswer{}, errors.New("no sealing key is presented, so the account cannot be sealed to the node: the enrolment is not spent") } digest := a.KeysDigest() - e, err := h.spend(a.Node, digest) + e, err := h.spend(a.Node, digest, a.OverlayKey) if err != nil { return delivery.RedeemAnswer{}, err } - if err := h.place(e, a); err != nil { - return delivery.RedeemAnswer{}, err - } account, err := h.Vault.Account(e.Node, a.SealingKey, digest) if err != nil { return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not give the account of %s (call redeem again with the same keys): %w", e.Node, err) } - cert, err := h.Vault.SignCertificate(e.Node, a.ServingKey) + cert, err := h.Vault.SignCertificate(e.Node, delivery.JoiningKeys{Identity: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey}) if err != nil { return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) } @@ -76,7 +74,7 @@ func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer // spend moves the key from live to redeeming with the presented keys' digest, by compare-and-set, before its expiry; at // redeeming it only checks the digest. A key that moved meanwhile is read again once. -func (h *Holder) spend(node, digest string) (Enrolment, error) { +func (h *Holder) spend(node, digest, overlay string) (Enrolment, error) { for attempt := 0; attempt < 3; attempt++ { row, found, err := h.Store.Get(NodeKey(node)) if err != nil { @@ -89,6 +87,16 @@ func (h *Holder) spend(node, digest string) (Enrolment, error) { if err != nil { return Enrolment{}, err } + // The tunnel key presented is the one the operator approved at issue, which placement and the hub already hold. + if overlay != e.OverlayKey { + return Enrolment{}, fmt.Errorf("the tunnel key presented is not the one %s's enrolment was issued for: the enrolment is not spent", node) + } + // The node entered placement on the operator's yes; the vault reads its address there. + if placements, err := h.placements(); err != nil { + return Enrolment{}, fmt.Errorf("placement cannot be read (call redeem again with the same keys): %w", err) + } else if p, ok := placements[node]; !ok || p.Decided.Address != e.Address { + return Enrolment{}, fmt.Errorf("placement does not hold %s at %s yet (call redeem again with the same keys)", node, e.Address) + } switch e.Stage { case StageRedeeming: if e.KeysDigest != digest { @@ -116,28 +124,3 @@ func (h *Holder) spend(node, digest string) (Enrolment, error) { } return Enrolment{}, fmt.Errorf("the enrolment of %s kept changing: call redeem again", node) } - -// place writes the node into placement with the issue's warrant, unless placement holds it at the approved address already. -func (h *Holder) place(e Enrolment, a delivery.Redeem) error { - placements, err := h.placements() - if err != nil { - return fmt.Errorf("placement cannot be read (call redeem again with the same keys): %w", err) - } - if p, ok := placements[e.Node]; ok && p.Decided.Address == e.Address { - return nil - } - if h.AddNode == nil || h.WarrantOf == nil { - return errors.New("no route to mesh-placement") - } - warrant, err := h.WarrantOf(e.Ask) - if err != nil { - return fmt.Errorf("the operator's warrant for %s cannot be read from the router (call redeem again with the same keys): %w", e.Ask, err) - } - change := delivery.PlacementProposalChange{Op: delivery.OpAddNode, Node: e.Node, Decided: &delivery.NodeDecided{Address: e.Address}, - Keys: &delivery.JoiningKeys{Identity: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey}} - if err := h.AddNode(AddNode{Why: "enrolment of " + e.Node + ": " + e.Why, Change: change, - Warrant: delivery.AddNodeWarrant{Node: e.Node, Address: e.Address, Ask: e.Ask, Warrant: warrant}}); err != nil { - return fmt.Errorf("%w (call redeem again with the same keys)", err) - } - return nil -} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go index 81599ff..d49cb5d 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/ed25519" "encoding/base64" - "slices" "testing" "time" @@ -13,9 +12,14 @@ import ( // redeemArgs are a node's redeem call with a proof signed by the key seeded, over the keys given. func redeemArgs(node string, seed byte, serving string) delivery.Redeem { + return redeemArgsWith(node, seed, serving, "overlay-"+node) +} + +// redeemArgsWith is redeemArgs presenting the tunnel key given. +func redeemArgsWith(node string, seed byte, serving, overlay string) delivery.Redeem { priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{seed}, ed25519.SeedSize)) a := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(priv.Public().(ed25519.PublicKey)), - OverlayKey: "overlay-" + node, SealingKey: "sealing-" + node, ServingKey: serving} + OverlayKey: overlay, SealingKey: "sealing-" + node, ServingKey: serving} a.Proof = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, delivery.RedeemProof(node, a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey))) return a } @@ -50,6 +54,14 @@ func redeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { _, _ = r.st.Put(NodeKey("ace"), e, 0) }, "waits for the operator's yes"}, {"no enrolment", "enrol.novox", redeemArgs("novox", 3, "s"), nil, "no enrolment of novox is open"}, + {"another tunnel key than issued", "enrol.ace", func() delivery.Redeem { + priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) + a := good + a.OverlayKey = "another-tunnel-key" + a.Proof = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, delivery.RedeemProof("ace", a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey))) + return a + }(), nil, "not the one ace's enrolment was issued for"}, + {"not in placement yet", "enrol.ace", good, func(r *rig) { delete(r.rd["mesh-delivery-assigner.placement"], "ace.node") }, "placement does not hold ace"}, } for _, c := range cases { r := newRig(t) @@ -78,13 +90,13 @@ func redeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { if e.Stage != StageRedeeming || e.KeysDigest != good.KeysDigest() || r.st.ttl[NodeKey("ace")] != 0 || r.st.ttl[AddressKey("10.9.0.2")] != 0 { t.Errorf("spent %+v, ttl %v/%v", e, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")]) } - if len(r.added) != 1 || r.added[0].Warrant != (delivery.AddNodeWarrant{Ask: "ask-ace", Node: "ace", Address: "10.9.0.2", Warrant: "ask-ace@2026-11-02T10:00:00Z"}) || - r.added[0].Change.Keys.Digest() != good.KeysDigest() || r.added[0].Change.Decided.Address != "10.9.0.2" || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { + // Redeem writes no placement: the yes did. It binds the identity keys, which the vault checks against the account's digest. + if len(r.added) != 0 || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { t.Errorf("placement %+v, answer %+v", r.added, ans) } - // Placement is written before the vault is asked: the vault reads the node's address from it. - if place, account := slices.Index(r.vault.calls, "place ace"), slices.Index(r.vault.calls, "account ace"); place < 0 || account < place { - t.Errorf("the vault was asked before placement held the node: %v", r.vault.calls) + // The certificate's call carries every key presented, which the vault checks against the account's digest. + if r.vault.cert.Digest() != good.KeysDigest() { + t.Errorf("the certificate was asked with %+v", r.vault.cert) } r.loaded = false if _, err := r.h.Redeem("enrol.ace", good); err == nil { @@ -113,8 +125,8 @@ func aRedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { if err != nil || got != want { t.Errorf("%s: %v, answer %+v, want %+v", stop, err, got, want) } - if stop == "sign ace" && len(r.added) != 1 { - t.Errorf("placement was written %d times", len(r.added)) + if len(r.added) != 0 { + t.Errorf("redeem wrote placement %d times", len(r.added)) } } // Placement that already holds the node at the approved address is not written again. diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index 75782fd..0fa1f58 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -13,13 +13,16 @@ import ( // seed writes an open enrolment of node at stage, with its address key, as the holder would have. func (r *rig) seed(node, address string, stage string) { - e := Enrolment{Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, Stage: stage, - Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour)} + e := Enrolment{Enrolment: delivery.Enrolment{Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, + Stage: stage, Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour)}, OverlayKey: "overlay-" + node} if stage == StageRedeeming { e.KeysDigest = delivery.KeysDigest("pk", "ov", "se", "sv") } _, _ = r.st.Put(NodeKey(node), e, 0) _, _ = r.st.Put(AddressKey(address), Reservation{Node: node}, 0) + if stage != StageAsked { // the operator's yes placed it, with its tunnel key + r.rd["mesh-delivery-assigner.placement"][node+".node"] = `{"v":1,"node":"` + node + `","decided":{"address":"` + address + `"},"modules":[]}` + } } // 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, the fill and whether issuing is on; diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go index 23a73dc..3365c65 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -50,22 +50,24 @@ func Tools(h *Holder) []stdio.Tool { "expiry (one hour unless given, at most 24 hours). Refused at once, naming every finding, for a name that is no node name " + "or that a node record, a placement key or an open enrolment holds, and for every call before the controller's lease is " + "written to a holder that is no controller. Otherwise it chooses the lowest free private-network address, reserves it, " + - "writes the enrolment at stage asked and asks the operator on a channel that proves who answered; only their yes makes " + - "the token (the vault makes it, this seat never holds it). The caller is the one the runtime proves.", + "writes the enrolment at stage asked and asks the operator on a channel that proves who answered; only their yes adds the " + + "node to placement with the machine's tunnel key (overlay_key, as `nox-mesh-host key` prints it there), so the hub peers " + + "it, and then makes the token, which carries the first tunnel (the vault makes it, this seat never holds it). The caller is the one the runtime proves.", Input: obj(map[string]any{"node": str("the new machine's name"), "why": str("why this machine joins"), - "expires": str("how long the token lasts, e.g. 1h (default) up to 24h")}, "node", "why"), + "expires": str("how long the token lasts, e.g. 1h (default) up to 24h"), + "overlay_key": str("the new machine's tunnel public key, as `nox-mesh-host key` prints it there")}, "node", "why", "overlay_key"), Run: func(a map[string]any) (any, error) { lasts, err := lastsOf(arg(a, "expires")) if err != nil { return nil, err } - return h.Issue(stdio.CallerOf(a), arg(a, "node"), arg(a, "why"), lasts) + return h.Issue(stdio.CallerOf(a), arg(a, "node"), arg(a, "why"), arg(a, "overlay_key"), lasts) }}, {Name: RedeemTool, Description: "The joining machine's one call (novox/hq ADR 0587 §5, §6), answered only for a caller the runtime proves to be " + "enrol.: its node-engine presents the public halves of the keys it made where it runs and a proof signed with its " + - "identity key. The token is spent (once, with those keys), the vault gives the node's account and signs its certificate, " + - "the node is written into placement on the issue's warrant, and the answer is given once the bus node loaded node., " + + "identity key, and the tunnel key it was issued for. The token is spent (once, with those keys) once placement holds the " + + "node (placed on the operator's yes), the vault gives the node's account and signs its certificate, and the answer is given once the bus node loaded node., " + "signed by the vault. Called again with the same keys it finishes what is not done and answers the same.", Input: obj(map[string]any{"node": str("the machine's name"), "public_key": str("its identity key, base64"), "overlay_key": str("its private-network key"), "sealing_key": str("its sealing key, base64"), @@ -89,8 +91,8 @@ func Tools(h *Holder) []stdio.Tool { Run: func(a map[string]any) (any, error) { return h.Confirm(stdio.CallerOf(a), arg(a, "node")) }}, {Name: RevokeTool, Description: "End an enrolment at any stage (novox/hq ADR 0587 §5): at asked it withdraws the ask; at live or redeeming it deletes " + - "the key and its address key, and at redeeming the vault retires an account it already minted (a node already written into " + - "placement stays there). It only takes access away, so it needs no Approve; it is recorded as an act done by hand. Says " + + "the key and its address key, and at redeeming the vault retires an account it already minted; a node the yes placed " + + "and that never confirmed is taken off placement again. It only takes access away, so it needs no Approve; it is recorded as an act done by hand. Says " + "enrolment-revoked; the vault then drops the token. A machine stuck at redeeming is ended by this. A revoke that stopped is " + "finished by calling it again.", Input: obj(map[string]any{"node": str("the machine"), "why": str("why, kept with the act")}, "node", "why"), diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go index 8ef9ff9..0ebbebb 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go @@ -40,8 +40,8 @@ func answer[T any](raw json.RawMessage, err error) (T, error) { } func (v vaultOver) Token(node string, expires time.Time, t delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { - // The seat's tunnel takes these three fields and no other (catalogue's enrolment-token). - tunnel := map[string]any{"hub_key": t.HubKey, "hub_endpoint": t.HubEndpoint, "address": t.Address} + // The whole first tunnel the token carries (delivery.TokenTunnel, catalogue's enrolment-token). + tunnel := map[string]any{"hub_key": t.HubKey, "hub_endpoint": t.HubEndpoint, "address": t.Address, "key": t.Key, "range": t.Range} return answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "expires": expires.UTC().Format(time.RFC3339), "tunnel": tunnel})) } @@ -74,8 +74,9 @@ func (v vaultOver) Retire(node string) error { return err } -func (v vaultOver) SignCertificate(node, servingKey string) (delivery.SignAnswer, error) { - return answer[delivery.SignAnswer](v.call(vaultSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node, "serving_key": servingKey})) +func (v vaultOver) SignCertificate(node string, k delivery.JoiningKeys) (delivery.SignAnswer, error) { + return answer[delivery.SignAnswer](v.call(vaultSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node, + "public_key": k.Identity, "overlay_key": k.Overlay, "sealing_key": k.Sealing, "serving_key": k.Serving})) } func (v vaultOver) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { From d1a256b42958230ab09c6feb8750571a39ce0703 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:15:12 +0100 Subject: [PATCH 13/20] mesh-grants: give the enrolment fixture its tunnel key, which an enrolment now requires mesh-sdk 22c03af (feat/l3-enrol) requires overlay-key on every enrolment; the fixture of the enrolment-user subtest carries one. (issue 774, review round 1b) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-grants/grants_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index d99bcee..09dc244 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -2425,7 +2426,8 @@ func TestGrants(t *testing.T) { issued := time.Date(2026, 11, 2, 1, 0, 0, 0, time.UTC) for node, stage := range map[string]string{"lab": delivery.EnrolmentLive, "pi": delivery.EnrolmentAsked} { raw, err := delivery.EncodeEnrolment(delivery.Enrolment{Node: node, Address: "10.0.0.7", Why: "joins", - Caller: "a.mesh-enrolment", Stage: stage, Issued: issued, Expires: issued.Add(time.Hour)}) + Caller: "a.mesh-enrolment", Stage: stage, Issued: issued, Expires: issued.Add(time.Hour), + OverlayKey: base64.StdEncoding.EncodeToString(make([]byte, 32))}) if err != nil { t.Fatal(err) } From cbcbb9644b49b4427c4d1f99e2e49a3cfbf24686 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:16:30 +0100 Subject: [PATCH 14/20] mesh-enrolment: keep the tunnel key in delivery.Enrolment's own field mesh-sdk 22c03af carries the machine's tunnel key on the enrolment (OverlayKey, overlay-key, required at every stage), so the holder's wrapper that kept it beside the contract is gone; the wire is unchanged. Test fixtures give the seeded enrolments a real tunnel key, which the contract now requires. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/enrolment.go | 40 +++---------------- .../cmd/mesh-enrolment/holder.go | 3 +- .../cmd/mesh-enrolment/issue.go | 4 +- .../cmd/mesh-enrolment/redeem_test.go | 2 +- .../cmd/mesh-enrolment/tokens_test.go | 4 +- 5 files changed, 13 insertions(+), 40 deletions(-) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go index 2b7a106..f492b5d 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go @@ -50,18 +50,11 @@ const ( StageRedeeming = delivery.EnrolmentRedeeming ) -// Reservation is the seat's state contract for an address key (delivery.EnrolmentAddress). -type Reservation = delivery.EnrolmentAddress - -// Enrolment is a node key: the seat's state contract (delivery.Enrolment) and the machine's tunnel key given at issue, which -// the node enters placement with on the operator's yes and which redeem must present again (the operator's decision of -// 2026-11-02 01:05). The contract does not carry the tunnel key yet: it is kept as one more member of the same JSON object, -// which every reader of the contract decodes as before (delivery's decode ignores a member it does not know). Once -// delivery.Enrolment carries `overlay-key` this wrapper goes. -type Enrolment struct { - delivery.Enrolment - OverlayKey string `json:"overlay-key,omitempty"` -} +// Enrolment and Reservation are the seat's state contract (delivery.Enrolment, delivery.EnrolmentAddress). +type ( + Enrolment = delivery.Enrolment + Reservation = delivery.EnrolmentAddress +) const ( nodePrefix = "node." @@ -85,33 +78,12 @@ func CheckNode(node string) error { return nil } -func decodeEnrolment(raw []byte) (Enrolment, error) { - e, err := delivery.DecodeEnrolment(raw) - if err != nil { - return Enrolment{}, err - } - var tunnel struct { - OverlayKey string `json:"overlay-key"` - } - _ = json.Unmarshal(raw, &tunnel) - return Enrolment{Enrolment: e, OverlayKey: tunnel.OverlayKey}, nil -} +func decodeEnrolment(raw []byte) (Enrolment, error) { return delivery.DecodeEnrolment(raw) } // Encode is a value of the bucket as the seat's contract writes it: an Enrolment or a Reservation checked and versioned. func Encode(v any) (json.RawMessage, error) { switch x := v.(type) { case Enrolment: - raw, err := delivery.EncodeEnrolment(x.Enrolment) - if err != nil || x.OverlayKey == "" { - return raw, err - } - var obj map[string]json.RawMessage - if err := json.Unmarshal(raw, &obj); err != nil { - return nil, err - } - obj["overlay-key"], _ = json.Marshal(x.OverlayKey) - return json.Marshal(obj) - case delivery.Enrolment: return delivery.EncodeEnrolment(x) case Reservation: return delivery.EncodeEnrolmentAddress(x.Node) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go index a6ec725..47b763e 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -149,7 +149,8 @@ func (h *Holder) warrantOf(e Enrolment) (delivery.AddNodeWarrant, error) { return delivery.AddNodeWarrant{Node: e.Node, Address: e.Address, Ask: e.Ask, Warrant: warrant}, nil } -// checkTunnelKey says a tunnel key is a WireGuard public key, as `nox-mesh-host key` prints it: 32 bytes in standard base64. +// checkTunnelKey says a tunnel key is a WireGuard public key, as `nox-mesh-host key` prints it: 32 bytes in standard base64 +// (as delivery.Enrolment requires it too); said here so issue names it among its findings. func checkTunnelKey(k string) error { raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(k)) if err != nil || len(raw) != 32 { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index bfb5d9b..1ddc577 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -98,8 +98,8 @@ func (h *Holder) Issue(caller, node, why, overlayKey string, lasts time.Duration return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %s", node, strings.Join(findings, "; ")) } now := h.now() - e := Enrolment{Enrolment: delivery.Enrolment{Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, - Issued: now, Expires: now.Add(lasts)}, OverlayKey: overlayKey} + e := Enrolment{Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, Issued: now, Expires: now.Add(lasts), + OverlayKey: overlayKey} e.Ask = "enrol-" + node + "-" + now.UTC().Format("20060102t150405") skip := map[string]bool{} for { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go index d49cb5d..e8b5b17 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -12,7 +12,7 @@ import ( // redeemArgs are a node's redeem call with a proof signed by the key seeded, over the keys given. func redeemArgs(node string, seed byte, serving string) delivery.Redeem { - return redeemArgsWith(node, seed, serving, "overlay-"+node) + return redeemArgsWith(node, seed, serving, tunnelKey) } // redeemArgsWith is redeemArgs presenting the tunnel key given. diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index 0fa1f58..00a691c 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -13,8 +13,8 @@ import ( // seed writes an open enrolment of node at stage, with its address key, as the holder would have. func (r *rig) seed(node, address string, stage string) { - e := Enrolment{Enrolment: delivery.Enrolment{Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, - Stage: stage, Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour)}, OverlayKey: "overlay-" + node} + e := Enrolment{Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, + Stage: stage, Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour), OverlayKey: tunnelKey} if stage == StageRedeeming { e.KeysDigest = delivery.KeysDigest("pk", "ov", "se", "sv") } From 44fee9eb0e65529a0e80e1b7d2f74dca7fa0ac96 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:27:45 +0100 Subject: [PATCH 15/20] mesh-enrolment: end an enrolment that expired unseen from its address key, taking the node out of placement ADR 0589 clause 6: a placed but unconfirmed node is always taken out again, on the warrant of its ask. The address key now names the ask and the issue's time (delivery.EnrolmentAddress, mesh-sdk c7130e3), written just before the ask is raised and with no expiry of its own, so it outlives the enrolment key; the holder deletes it at every end. An expiry, seen or not, ends the enrolment in this order, each step safe to repeat: remove-node on the ask's warrant (only when placement holds the node), enrolment-expired naming the rebuilt issued event (delivery.EnrolmentIssuedID), the vault's drop of the token, and the address key last. Reconcile finds every address key that names an ask and whose enrolment key is gone (expired while the holder was down, before its watch caught up, or an end that stopped) and ends it; it runs once the watch has caught up, on every expiry, and again with a growing wait while a step fails. An address reserved by an issue under way (no ask yet) is left alone. Tests: rows in the end table (expired unseen; the holder down with a drop that fails once; a reservation under way) and the integration test's expiry case (the address key goes; a holder that was down unplaces on its pass); no new function. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../mesh-enrolment/cmd/mesh-enrolment/end.go | 95 +++++++++++++++---- .../cmd/mesh-enrolment/end_test.go | 48 ++++++++-- .../cmd/mesh-enrolment/enrolment.go | 2 +- .../cmd/mesh-enrolment/integration_test.go | 40 +++++++- .../cmd/mesh-enrolment/issue.go | 8 +- .../cmd/mesh-enrolment/issue_test.go | 8 +- .../cmd/mesh-enrolment/redeem.go | 6 +- .../cmd/mesh-enrolment/redeem_test.go | 6 +- .../mesh-enrolment/cmd/mesh-enrolment/run.go | 13 ++- .../cmd/mesh-enrolment/tokens_test.go | 2 +- 10 files changed, 189 insertions(+), 39 deletions(-) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go index 7d2584e..1a00b8b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "fmt" "strings" @@ -86,25 +87,83 @@ func (h *Holder) Heard(op, key string, value []byte) error { case stdio.OpExpire: e, known := h.seen[node] delete(h.seen, node) - if !known { - // The key expired before the watch handed its content over (a holder started after it was written, before its - // catch-up): the enrolment-issued event cannot be named, and an expiry naming none is no expiry (the contract - // refuses it). Nothing is said; the token is dropped at once, as an ending event would have it dropped. - h.logf("the enrolment of %s expired before its content was seen, so enrolment-expired cannot name its issued event and is not said; its token is dropped", node) - if err := h.Vault.Drop(node); err != nil { - return fmt.Errorf("the token of %s, whose enrolment expired unseen, is not dropped (offered again): %w", node, err) + if known { + return h.endExpired(e) + } + // The key expired before the watch handed its content over: its address key still names its ask and issue (ADR 0589 §6). + return h.reconcileLocked() + } + return nil +} + +// Reconcile ends every enrolment whose key is gone while its address key still names its ask: one that expired while the +// holder was down, or before its watch caught up, or whose end stopped midway (ADR 0589 §6). Run at start, after each +// catch-up and on every expiry; a step that fails is tried again on the next pass. +func (h *Holder) Reconcile() error { + h.mu.Lock() + defer h.mu.Unlock() + return h.reconcileLocked() +} + +func (h *Holder) reconcileLocked() error { + keys, err := h.Store.Keys() + if err != nil { + return fmt.Errorf("the enrolments cannot be listed: %w", err) + } + open := map[string]bool{} + for _, k := range keys { + if n, ok := strings.CutPrefix(k, "node."); ok { + open[n] = true + } + } + var errs []error + for _, k := range keys { + address, ok := strings.CutPrefix(k, "address.") + if !ok { + continue + } + row, found, err := h.Store.Get(k) + if err != nil || !found { + continue + } + r, err := delivery.DecodeEnrolmentAddress(row.Value) + // An address with no ask is reserved by an issue under way: its enrolment key is written next, then the ask. + if err != nil || r.Ask == "" || open[r.Node] { + continue + } + h.logf("the enrolment of %s (ask %s) is gone and its address %s is still reserved: it expired unseen, and is ended now", r.Node, r.Ask, address) + if err := h.endExpired(Enrolment{Node: r.Node, Address: address, Ask: r.Ask, Issued: r.Issued}); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// endExpired ends an enrolment whose key expired, in this order, each step safe to do again: the node, placed on the yes and +// never confirmed, is taken out of placement on the warrant of its ask (only when placement holds it); enrolment-expired names +// its issued event (said under one id, so said again is said once); the vault drops the token; the address key goes last, so +// a step that failed is found and done on the next pass (ADR 0589 §6). +func (h *Holder) endExpired(e Enrolment) error { + if err := h.unplace(e, "expired"); err != nil { + return fmt.Errorf("%s, whose enrolment expired, is not taken out of placement (tried again on the next pass): %w", e.Node, err) + } + body, err := delivery.EncodeEnrolmentExpiry(delivery.EnrolmentExpiry{Node: e.Node, IssuedID: delivery.EnrolmentIssuedID(e.Node, e.Issued)}) + if err != nil { + return err + } + if err := h.say(EventExpired, json.RawMessage(body), eventID(EventExpired, e)); err != nil { + return fmt.Errorf("enrolment-expired of %s cannot be said (tried again on the next pass): %w", e.Node, err) + } + if err := h.Vault.Drop(e.Node); err != nil { + return fmt.Errorf("the token of %s, whose enrolment expired, is not dropped (tried again on the next pass): %w", e.Node, err) + } + if row, found, err := h.Store.Get(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s cannot be read (tried again on the next pass): %w", e.Address, err) + } else if found { + if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node == e.Node && r.Ask == e.Ask { + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s is not deleted (tried again on the next pass): %w", e.Address, err) } - return nil - } - if err := h.unplace(e, "expired"); err != nil { - return fmt.Errorf("%s, whose enrolment expired, is not taken off placement (offered again): %w", node, err) - } - body, err := delivery.EncodeEnrolmentExpiry(delivery.EnrolmentExpiry{Node: node, IssuedID: eventID(EventIssued, e)}) - if err != nil { - return err - } - if err := h.say(EventExpired, json.RawMessage(body), eventID(EventExpired, e)); err != nil { - return fmt.Errorf("enrolment-expired of %s cannot be said (offered again): %w", node, err) } } return nil diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go index e812414..40f1d71 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -7,6 +7,7 @@ import ( "slices" "strings" "testing" + "time" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -53,13 +54,48 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { } last := r.said[len(r.said)-1] x, err := delivery.DecodeEnrolmentExpiry(last.Body) - if err != nil || last.Event != EventExpired || x.Node != "novox" || !strings.HasPrefix(x.IssuedID, "enrolment-issued.novox.") { - t.Errorf("said %s %s: %+v, %v", last.Event, last.Body, x, err) + if err != nil || last.Event != EventExpired || x.Node != "novox" || !strings.HasPrefix(x.IssuedID, "enrolment-issued.novox.") || + !slices.Contains(r.vault.calls, "drop novox") || r.has(AddressKey("10.9.0.4")) { + t.Errorf("said %s %s: %+v, %v; vault %v, keys %v", last.Event, last.Body, x, err, r.vault.calls, r.keys()) } - // A key that expired before its content was seen names no issued event: nothing is said, and the token is dropped. - said := len(r.said) - if err := r.h.Heard(stdio.OpExpire, NodeKey("unseen"), nil); err != nil || len(r.said) != said || !slices.Contains(r.vault.calls, "drop unseen") { - t.Errorf("an unseen expiry: %v, said %v, vault %v", err, r.events(), r.vault.calls) + // ADR 0589 §6: an enrolment whose key expired unseen (before the watch handed it over, or while the holder was down) is + // found by its address key, which names its ask and issue: it leaves placement on that ask's warrant, enrolment-expired + // names its rebuilt issued event, the token is dropped and the address key goes last. A step that fails is done again on + // the next pass, the address key kept until then. + for _, c := range []struct { + name string + node string + fail string + passS func(r *rig) error + }{ + {"expired unseen", "unseen", "", func(r *rig) error { return r.h.Heard(stdio.OpExpire, NodeKey("unseen"), nil) }}, + {"the holder was down, and its drop failed once", "down", "drop down", func(r *rig) error { return r.h.Reconcile() }}, + } { + r.seed(c.node, "10.9.0.9", StageLive) + issued := r.now.Add(-time.Minute) + _ = r.st.Delete(NodeKey(c.node)) // the bus took the key; the watch never handed its content over + if c.fail != "" { + r.vault.fail[c.fail] = true + if err := c.passS(r); err == nil || !r.has(AddressKey("10.9.0.9")) { + t.Errorf("%s: a failed drop ended the pass (%v) or took the address key", c.name, err) + } + } + if err := c.passS(r); err != nil { + t.Fatalf("%s: %v", c.name, err) + } + last := r.said[len(r.said)-1] + x, err := delivery.DecodeEnrolmentExpiry(last.Body) + _, placed := r.rd["mesh-delivery-assigner.placement"][c.node+".node"] + removed := r.added[len(r.added)-1] + if err != nil || x.IssuedID != delivery.EnrolmentIssuedID(c.node, issued) || placed || removed.Change.Op != delivery.OpRemoveNode || + removed.Change.Node != c.node || removed.Warrant.Ask != "ask-"+c.node || !slices.Contains(r.vault.calls, "drop "+c.node) || r.has(AddressKey("10.9.0.9")) { + t.Errorf("%s: said %s (%v), placed %v, last placement call %+v, vault %v, keys %v", c.name, last.Body, err, placed, removed, r.vault.calls, r.keys()) + } + } + // An address reserved by an issue under way (no ask yet) is never taken for an expired enrolment. + _, _ = r.st.Put(AddressKey("10.9.0.10"), Reservation{Node: "busy"}, 0) + if err := r.h.Reconcile(); err != nil || !r.has(AddressKey("10.9.0.10")) { + t.Errorf("a reservation under way was ended: %v", err) } } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go index f492b5d..9994ad5 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go @@ -86,7 +86,7 @@ func Encode(v any) (json.RawMessage, error) { case Enrolment: return delivery.EncodeEnrolment(x) case Reservation: - return delivery.EncodeEnrolmentAddress(x.Node) + return delivery.EncodeEnrolmentAddress(x.Node, x.Ask, x.Issued) } return json.Marshal(v) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go index ecacd45..b351880 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -16,6 +16,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -283,6 +284,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { var usersMu sync.Mutex users := map[string]bool{} hasUser := func(u string) bool { usersMu.Lock(); defer usersMu.Unlock(); return users[u] } + var down atomic.Bool // the holder is down: its watch hands nothing over w, err := enrolments.WatchAll(ctx) if err != nil { t.Fatal(err) @@ -294,7 +296,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { continue } node, ok := strings.CutPrefix(e.Key(), "node.") - if !ok { + if !ok || down.Load() { continue } if e.Operation() == jetstream.KeyValuePut { @@ -411,4 +413,40 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { if _, err := placement.Get(ctx, "g14.node"); err == nil { t.Error("g14 stays in placement after its enrolment expired") } + // Its address key had no expiry of its own: the holder took it last, after the drop (ADR 0589 §6). + waitFor("the expired enrolment's address key is deleted", func() bool { keys, _ := store.Keys(); return len(keys) == 0 }) + + // The holder was down when an enrolment expired: a holder started later finds its address key, which names its ask, and + // ends it the same way (here: the key deleted under the holder, then a pass as at start). + b, err := h.Issue("person.operator", "novox", "a third machine", tunnelKey, 0) + if err != nil { + t.Fatal(err) + } + if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: b.Ask}); err != nil { + t.Fatal(err) + } + if _, err := placement.Get(ctx, "novox.node"); err != nil { + t.Fatalf("novox was not placed on the yes: %v", err) + } + down.Store(true) + if err := enrolments.Purge(ctx, NodeKey("novox")); err != nil { // the bus took the key while no holder watched + t.Fatal(err) + } + waitFor("the key is gone", func() bool { _, found, _ := store.Get(NodeKey("novox")); return !found }) + if _, err := placement.Get(ctx, "novox.node"); err != nil { + t.Fatalf("novox left placement while no holder ran: %v", err) + } + if err := h.Reconcile(); err != nil { + t.Fatal(err) + } + if _, err := placement.Get(ctx, "novox.node"); err == nil { + t.Error("novox, expired while the holder was down, stays in placement") + } + waitFor("the token of novox is dropped and its address key goes", func() bool { + vault.mu.Lock() + dropped := slices.Contains(vault.calls, "drop novox") + vault.mu.Unlock() + keys, _ := store.Keys() + return dropped && len(keys) == 0 + }) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index 1ddc577..e52f3e8 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -196,6 +196,11 @@ func (h *Holder) raise(e Enrolment) error { if err != nil { return err } + // The address key names the ask and the issue's time from now, with no expiry of its own: it outlives the enrolment key, + // so an enrolment that expires unseen can still be taken out of placement on its ask's warrant (ADR 0589 §6). + if _, err := h.Store.Put(AddressKey(e.Address), Reservation{Node: e.Node, Ask: e.Ask, Issued: e.Issued}, 0); err != nil { + return fmt.Errorf("the address key %s cannot name the ask %s, so the operator is not asked: %w", e.Address, e.Ask, err) + } if err := h.Asker.Ask(ask); err != nil { if cerr := h.Asker.Cancel(ask.ID); cerr != nil { h.logf("the ask %s that could not be published was not taken back at the router: %v", ask.ID, cerr) @@ -265,9 +270,6 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { if _, err := h.Store.Put(NodeKey(e.Node), e, ttl); err != nil { return fmt.Errorf("the enrolment of %s cannot be moved to live: %w", e.Node, err) } - if _, err := h.Store.Put(AddressKey(e.Address), Reservation{Node: e.Node}, ttl); err != nil { - return fmt.Errorf("the address key %s cannot be given its expiry: %w", e.Address, err) - } return h.sayEvent(EventIssued, e, "") } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go index f622fb7..7c59eaf 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go @@ -99,10 +99,16 @@ func twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { t.Fatal(err) } live, _ := r.enrolment("ace") - if live.Stage != StageLive || r.st.ttl[NodeKey("ace")] != time.Hour || r.st.ttl[AddressKey(live.Address)] != time.Hour || + if live.Stage != StageLive || r.st.ttl[NodeKey("ace")] != time.Hour || r.st.ttl[AddressKey(live.Address)] != 0 || len(r.warr) != 1 || r.warr[0] != e.Ask || r.events()[0] != EventIssued { t.Errorf("live %+v, ttl %v/%v, warrants %v, events %v", live, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey(live.Address)], r.warr, r.events()) } + // The address key names the ask and the issue, with no expiry of its own, so it outlives the enrolment key (ADR 0589 §6). + if row, _, _ := r.st.Get(AddressKey(live.Address)); true { + if res, err := delivery.DecodeEnrolmentAddress(row.Value); err != nil || res.Ask != e.Ask || !res.Issued.Equal(live.Issued) { + t.Errorf("the address key %s: %+v, %v", row.Value, res, err) + } + } if got, want := r.vault.tunnels["ace"], (delivery.TokenTunnel{Key: tunnelKey, Address: live.Address + "/32", Range: "10.9.0.0/24", HubKey: "hub-tunnel-key", HubEndpoint: "192.0.2.1:51820"}); got != want { t.Errorf("the token's tunnel %+v, want %+v", got, want) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go index 93a8355..7ee0b09 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -116,10 +116,8 @@ func (h *Holder) spend(node, digest, overlay string) (Enrolment, error) { } return Enrolment{}, fmt.Errorf("the enrolment of %s cannot be spent: %w", node, err) } - // The move to redeeming rewrites both keys without an expiry, so a redeem under way never loses its key (§5). - if _, err := h.Store.Put(AddressKey(e.Address), Reservation{Node: node}, 0); err != nil { - return Enrolment{}, fmt.Errorf("the address key %s cannot be kept (call redeem again with the same keys): %w", e.Address, err) - } + // The move to redeeming rewrites the key without an expiry, so a redeem under way never loses it (§5); the address key + // never had one. return e, nil } return Enrolment{}, fmt.Errorf("the enrolment of %s kept changing: call redeem again", node) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go index e8b5b17..6edebd4 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -81,14 +81,14 @@ func redeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { } r := newRig(t) r.seed("ace", "10.9.0.2", StageLive) - r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")] = time.Hour, time.Hour + r.st.ttl[NodeKey("ace")] = time.Hour ans, err := r.h.Redeem("enrol.ace", good) if err != nil { t.Fatal(err) } e, _ := r.enrolment("ace") - if e.Stage != StageRedeeming || e.KeysDigest != good.KeysDigest() || r.st.ttl[NodeKey("ace")] != 0 || r.st.ttl[AddressKey("10.9.0.2")] != 0 { - t.Errorf("spent %+v, ttl %v/%v", e, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey("10.9.0.2")]) + if e.Stage != StageRedeeming || e.KeysDigest != good.KeysDigest() || r.st.ttl[NodeKey("ace")] != 0 { + t.Errorf("spent %+v, ttl %v", e, r.st.ttl[NodeKey("ace")]) } // Redeem writes no placement: the yes did. It binds the identity keys, which the vault checks against the account's digest. if len(r.added) != 0 || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/run.go b/modules/mesh-enrolment/cmd/mesh-enrolment/run.go index 07cd094..b23efc9 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/run.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/run.go @@ -10,7 +10,18 @@ import ( // markers), the durable consumer of its own ending events (the drop), and the operator's answers. Each is asked of the // runtime as soon as it serves, and again with a growing wait until the runtime takes it. func (h *Holder) Run(listen func() error) { - go stdio.State(State).Follow("", func(c stdio.StateChange) error { return h.Heard(c.Op, c.Key, c.Value) }) + go func() { + stdio.State(State).Follow("", func(c stdio.StateChange) error { + err := h.Heard(c.Op, c.Key, c.Value) + if err != nil && c.Op == stdio.OpExpire { + // An expiry whose end stopped is finished by a later pass (ADR 0589 §6). + go retry("finish the enrolments that expired", h.logf, h.Reconcile) + } + return err + }) + // Caught up: an enrolment that expired while the holder was down is ended now. + retry("end the enrolments that expired unseen", h.logf, h.Reconcile) + }() for _, event := range EndEvents { go func(key string) { <-stdio.Served() diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index 00a691c..f7e9c01 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -19,7 +19,7 @@ func (r *rig) seed(node, address string, stage string) { e.KeysDigest = delivery.KeysDigest("pk", "ov", "se", "sv") } _, _ = r.st.Put(NodeKey(node), e, 0) - _, _ = r.st.Put(AddressKey(address), Reservation{Node: node}, 0) + _, _ = r.st.Put(AddressKey(address), Reservation{Node: node, Ask: e.Ask, Issued: e.Issued}, 0) if stage != StageAsked { // the operator's yes placed it, with its tunnel key r.rd["mesh-delivery-assigner.placement"][node+".node"] = `{"v":1,"node":"` + node + `","decided":{"address":"` + address + `"},"modules":[]}` } From 795724f5ca6cfd8d9b3e2767733a71aa09796dad Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:30:04 +0100 Subject: [PATCH 16/20] mesh-enrolment: catch up only on an ask answered yes, and drop the address key first at confirm, decline and refused add ADR 0589 version 29, clause 6. At start and after a catch-up an address key is an expiry never heard only when its enrolment key is gone and the router records its ask answered yes; one declined or not answered yet is no expiry, and one not answered stays until its ask ends. A confirm, a decline and a refused add delete the address key before their other steps, so one stopped midway never leaves a key the catch-up would take for an expiry and unplace a node that joined. A refused add (mesh-placement's refusal of the proposal, not a call that did not reach it) now ends the enrolment as refused; a call that failed still waits for again. Tests: rows in the end table (after a confirm stopped before its enrolment key went, after a decline, after a refused add, a declined ask, an ask not answered: nothing unplaced) and the integration test's holder-was-down case reading the yes; no new function. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/confirm.go | 5 ++ .../mesh-enrolment/cmd/mesh-enrolment/end.go | 17 +++++ .../cmd/mesh-enrolment/end_test.go | 67 +++++++++++++++++++ .../cmd/mesh-enrolment/integration_test.go | 1 + .../cmd/mesh-enrolment/issue.go | 33 +++++++++ .../cmd/mesh-enrolment/kit_test.go | 20 ++++++ 6 files changed, 143 insertions(+) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go index ee8977a..9324bcd 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go @@ -32,6 +32,11 @@ func (h *Holder) Confirm(caller, node string) (map[string]any, error) { if e.Stage != StageRedeeming { return nil, fmt.Errorf("the enrolment of %s is %s: it is confirmed after redeem", own, e.Stage) } + // The address key goes first (ADR 0589 §6): a confirm stopped midway never leaves one the catch-up would take for an + // expiry and unplace a node that joined. + if err := h.dropAddress(e); err != nil { + return nil, fmt.Errorf("%w (call confirm again)", err) + } if err := h.sayEvent(EventEnrolled, e, ""); err != nil { return nil, fmt.Errorf("node-enrolled of %s cannot be said, so nothing is deleted (call confirm again): %w", own, err) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go index 1a00b8b..47f045a 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -131,6 +131,14 @@ func (h *Holder) reconcileLocked() error { if err != nil || r.Ask == "" || open[r.Node] { continue } + // Only an ask answered yes made a token and placed the node (ADR 0589 §6): one declined, or not answered yet, is no + // expiry, and one not answered stays until its ask ends. + if yes, err := h.answeredYes(r.Ask); err != nil { + errs = append(errs, fmt.Errorf("the answer to %s cannot be read, so the address %s waits for the next pass: %w", r.Ask, address, err)) + continue + } else if !yes { + continue + } h.logf("the enrolment of %s (ask %s) is gone and its address %s is still reserved: it expired unseen, and is ended now", r.Node, r.Ask, address) if err := h.endExpired(Enrolment{Node: r.Node, Address: address, Ask: r.Ask, Issued: r.Issued}); err != nil { errs = append(errs, err) @@ -139,6 +147,15 @@ func (h *Holder) reconcileLocked() error { return errors.Join(errs...) } +// answeredYes reads the router's record of ask: whether the operator answered it yes. +func (h *Holder) answeredYes(ask string) (bool, error) { + _, w, err := h.whereIs(ask) + if err != nil { + return false, err + } + return w != nil && w.Option == "yes", nil +} + // endExpired ends an enrolment whose key expired, in this order, each step safe to do again: the node, placed on the yes and // never confirmed, is taken out of placement on the warrant of its ask (only when placement holds it); enrolment-expired names // its issued event (said under one id, so said again is said once); the vault drops the token; the address key goes last, so diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go index 40f1d71..a341180 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -2,6 +2,8 @@ package main import ( "encoding/json" + "errors" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" "reflect" "slices" @@ -72,6 +74,7 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { {"the holder was down, and its drop failed once", "down", "drop down", func(r *rig) error { return r.h.Reconcile() }}, } { r.seed(c.node, "10.9.0.9", StageLive) + r.asker.answer("ask-"+c.node, "yes") issued := r.now.Add(-time.Minute) _ = r.st.Delete(NodeKey(c.node)) // the bus took the key; the watch never handed its content over if c.fail != "" { @@ -92,6 +95,70 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { t.Errorf("%s: said %s (%v), placed %v, last placement call %+v, vault %v, keys %v", c.name, last.Body, err, placed, removed, r.vault.calls, r.keys()) } } + // ADR 0589 §6, the negative rows: an address key the catch-up finds with its enrolment key gone is no expiry when its ask + // was declined or is not answered yet, and none is left behind by a confirm, a decline or a refused add, which delete it + // first. In none of them is the node unplaced. + for _, c := range []struct { + name string + end func(r *rig, node string) + key bool // the address key stays (an ask not answered yet stays until its ask ends) + }{ + {"after a confirm stopped before the enrolment key went", func(r *rig, node string) { + e, _ := r.enrolment(node) + e.Stage, e.KeysDigest = StageRedeeming, delivery.KeysDigest("pk", "ov", "se", "sv") + _, _ = r.st.Put(NodeKey(node), e, 0) + r.asker.answer("ask-"+node, "yes") + r.st.failDelete[NodeKey(node)] = true // the confirm stops after the address key, before the enrolment key + _, _ = r.h.Confirm(NodeUser(node), "") + _ = r.st.Delete(NodeKey(node)) // and the key is gone by the next pass + }, false}, + {"after a decline", func(r *rig, node string) { + _ = r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: "ask-" + node}) + r.asker.answer("ask-"+node, "no") + }, false}, + {"after a refused add", func(r *rig, node string) { + delete(r.rd["mesh-delivery-assigner.placement"], node+".node") // the add is tried, and refused + r.placeErr = errors.New("mesh-placement did not take the node: the proposal is refused: change 1: an address taken") + _ = r.h.Warranted(yes, asks.Warrant{Ask: "ask-" + node}) + r.asker.answer("ask-"+node, "yes") + r.placeErr = nil + if !slices.Contains(r.events(), EventRefused) { + r.t.Errorf("a refused add did not end the enrolment: %v", r.events()) + } + }, false}, + {"a declined ask whose enrolment key is gone", func(r *rig, node string) { + r.asker.answer("ask-"+node, "no") + _ = r.st.Delete(NodeKey(node)) + }, true}, + {"an ask not answered yet", func(r *rig, node string) { _ = r.st.Delete(NodeKey(node)) }, true}, + } { + r := newRig(t) + stage := StageAsked + if strings.HasPrefix(c.name, "after a confirm") { + stage = StageLive + } + r.seed("joiner", "10.9.0.11", stage) + r.rd["mesh-delivery-assigner.placement"]["joiner.node"] = `{"v":1,"node":"joiner","decided":{"address":"10.9.0.11"},"modules":[]}` + c.end(r, "joiner") + removes := 0 + for _, a := range r.added { + if a.Change.Op == delivery.OpRemoveNode { + removes++ + } + } + if err := r.h.Reconcile(); err != nil { + t.Errorf("%s: %v", c.name, err) + } + after := 0 + for _, a := range r.added { + if a.Change.Op == delivery.OpRemoveNode { + after++ + } + } + if after != removes || r.has(AddressKey("10.9.0.11")) != c.key || slices.Contains(r.events(), EventExpired) { + t.Errorf("%s: the catch-up unplaced (%d to %d), address key %v, events %v", c.name, removes, after, r.has(AddressKey("10.9.0.11")), r.events()) + } + } // An address reserved by an issue under way (no ask yet) is never taken for an expired enrolment. _, _ = r.st.Put(AddressKey("10.9.0.10"), Reservation{Node: "busy"}, 0) if err := r.h.Reconcile(); err != nil || !r.has(AddressKey("10.9.0.10")) { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go index b351880..fe757eb 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -425,6 +425,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: b.Ask}); err != nil { t.Fatal(err) } + asker.answer(b.Ask, "yes") // the router's record of the answer, which the catch-up reads (ADR 0589 §6) if _, err := placement.Get(ctx, "novox.node"); err != nil { t.Fatalf("novox was not placed on the yes: %v", err) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index e52f3e8..597f542 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -260,6 +260,10 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { // The node enters placement now, with its tunnel key, on this warrant: the hub peers it before the machine redeems over // the tunnel (the operator's decision of 2026-11-02 01:05). Only then is the token made. if err := h.place(e, placements); err != nil { + if addRefused(err) { + // mesh-placement refused the node itself, not the call to it: the enrolment ends (ADR 0589 §6). + return h.refuse(e, "mesh-placement refused the node: "+err.Error()) + } return fmt.Errorf("the node %s is not added to placement, so no token is made (mesh-enrolment.again asks again): %w", e.Node, err) } if _, err := h.Vault.Token(e.Node, e.Expires, tunnel); err != nil { @@ -273,8 +277,37 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { return h.sayEvent(EventIssued, e, "") } +// addRefused says mesh-placement answered the add-node with a refusal of the proposal (the assigner's words, "the proposal is +// refused: …"), as against a call that did not reach it or was not answered, which `again` asks once more. +func addRefused(err error) bool { + return err != nil && strings.Contains(err.Error(), "the proposal is refused") +} + +// dropAddress deletes the enrolment's address key while it still names this node: the first step of a confirm, a decline and a +// refused add (ADR 0589 §6), so the catch-up never takes a key left by one of them for an expiry never heard. +func (h *Holder) dropAddress(e Enrolment) error { + row, found, err := h.Store.Get(AddressKey(e.Address)) + if err != nil { + return fmt.Errorf("the address key %s cannot be read, so the end of %s waits: %w", e.Address, e.Node, err) + } + if !found { + return nil + } + if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node != e.Node { + return nil // another enrolment holds the address now + } + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s is not deleted, so the end of %s waits: %w", e.Address, e.Node, err) + } + return nil +} + // refuse ends an enrolment at stage asked: both keys go and enrolment-refused carries the key's content (ADR 0587 §4). func (h *Holder) refuse(e Enrolment, reason string) error { + // The address key goes first (ADR 0589 §6): a refusal stopped midway never leaves one that looks like an expiry. + if err := h.dropAddress(e); err != nil { + return err + } // A yes whose token failed added the node already; a refusal takes it off again. if err := h.unplace(e, reason); err != nil { return fmt.Errorf("%s is not taken off placement, so its enrolment stays: %w", e.Node, err) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 45a17b6..4dbad2b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -158,6 +158,26 @@ type fakeAsker struct { mu sync.Mutex asked, cancelled []string err error + answers map[string]string // the router's record: the option each ask was answered with +} + +// answer records that the operator answered ask with option, as the router keeps it. +func (a *fakeAsker) answer(ask, option string) { + a.mu.Lock() + defer a.mu.Unlock() + if a.answers == nil { + a.answers = map[string]string{} + } + a.answers[ask] = option +} + +func (a *fakeAsker) Where(id string) (string, *asks.Warrant, error) { + a.mu.Lock() + defer a.mu.Unlock() + if o, ok := a.answers[id]; ok { + return "answered", &asks.Warrant{Ask: id, Option: o}, nil + } + return "", nil, nil // no record read: as a router whose record of the ask is gone } func (a *fakeAsker) Ask(k asks.Ask) error { From bde60130a8161d880673a8d5d5dd55ec61472919 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:48:53 +0100 Subject: [PATCH 17/20] The assigner's add-node test writes no lease: since the removal group the assigner reads none and its bed has no lease bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enrolment's issue waits for the lease write itself (ADR 0587 §4); the assigner takes an add-node on its warrant at any time (ADR 0567 §8). (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-delivery-assigner/integration_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1457c91..e871f34 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1134,9 +1134,9 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { t.Errorf("the silent node's verdict = %+v", failed.Verdict) } - // 774/S59 (ADR 0587 §7): once the lease is handed away, a joining node is written on the enrolment's warrant with no node to - // answer and no ask raised, and placement-changed carries that warrant. - b.putJSON(t, buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-grants","handed-by":"operator"}`)) + // 774/S59 (ADR 0587 §7): a joining node is written on the enrolment's warrant with no node to answer and no ask raised, + // and placement-changed carries that warrant. The assigner reads no lease (ADR 0567 §8): the enrolment's own issue waits + // for the lease write (ADR 0587 §4). for len(heard) > 0 { <-heard } From e6ee939bd46641d2b00fb6e58b8f6e7f0b61c2f1 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:48:53 +0100 Subject: [PATCH 18/20] =?UTF-8?q?The=20indexer's=20closed=20seat=20set=20c?= =?UTF-8?q?ounts=20fifty-eight=20with=20mesh-enrolment=20(ADR=200587=20?= =?UTF-8?q?=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-indexer/judge/rules_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-indexer/judge/rules_test.go b/modules/mesh-delivery-indexer/judge/rules_test.go index 5445bd7..1d6c599 100644 --- a/modules/mesh-delivery-indexer/judge/rules_test.go +++ b/modules/mesh-delivery-indexer/judge/rules_test.go @@ -5014,9 +5014,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1); // fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts // (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2); - // fifty-seven with mesh-grants (ADR 0582 clause 1). - if len(catalogue.Seats()) != 57 { - t.Errorf("the mesh defines %d seats rather than 57; the set is closed, so a change here is "+ + // fifty-seven with mesh-grants (ADR 0582 clause 1); fifty-eight with mesh-enrolment (novox/hq ADR 0587 §1). + if len(catalogue.Seats()) != 58 { + t.Errorf("the mesh defines %d seats rather than 58; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(catalogue.Seats()), seatNames()) } } From b13f65674b35bd06ec1c06b1cabe2d9b6772ac1b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:48:53 +0100 Subject: [PATCH 19/20] Pin mesh-sdk at dff0ce2, the enrolment group's rebased head, so every member builds the same contract (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 2 ++ modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 2 ++ modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 2 ++ modules/mesh-enrolment/go.mod | 2 +- modules/mesh-enrolment/go.sum | 48 ++++++++++++++------------- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 2 ++ 10 files changed, 38 insertions(+), 28 deletions(-) diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index def263e..4a1f553 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 60e9ab1..633697d 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavo git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 95009ea..236cb80 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 8a22cf0..cf2ce55 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmH git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index a74ba03..7d4da6f 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index e59eb50..b494e51 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMi git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-enrolment/go.mod b/modules/mesh-enrolment/go.mod index 10f8f09..52cdb8e 100644 --- a/modules/mesh-enrolment/go.mod +++ b/modules/mesh-enrolment/go.mod @@ -3,7 +3,7 @@ module mesh-enrolment go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-enrolment/go.sum b/modules/mesh-enrolment/go.sum index 984c03d..ce8dc67 100644 --- a/modules/mesh-enrolment/go.sum +++ b/modules/mesh-enrolment/go.sum @@ -1,29 +1,31 @@ -github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= -github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= -github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= -github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= -github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= -github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= -github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= -github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= -github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= -github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= -github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= -github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= -github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0= -golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index c069fd3..3f7dd28 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index adb6e38..df8d76d 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOH git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 63c8687166e6508f8213c930959db9e6edc69bec Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 02:07:36 +0100 Subject: [PATCH 20/20] Let a bus module's tools bundle declare what it invokes (issue 774) The judge refused nats's manifest for its invokes (seat:mesh-hand-acts.hand-act, which nats_bucket_expiry asks to record its hand act), and with one manifest refused the grants part holds the whole bus user list, freezing every build that adds bus permissions. The bundle asks through the node's tool runner, which buscompose grants every tool and seat verb, so the invokes are granted: the same condition that already lets the bundle declare emits (ADR 0357). ADR 0235's rule stands for a bus module with no bundle. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-indexer/judge/rules.go | 17 +++++---- .../mesh-delivery-indexer/judge/rules_test.go | 35 +++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-indexer/judge/rules.go b/modules/mesh-delivery-indexer/judge/rules.go index 37dd9f2..fa3ef9e 100644 --- a/modules/mesh-delivery-indexer/judge/rules.go +++ b/modules/mesh-delivery-indexer/judge/rules.go @@ -2450,19 +2450,24 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M // holding mesh-broker is granted the snapshot API and nothing else, so anything it declared to say // or hear on the bus would be granted nothing — refused here rather than silently dropped. // - // **But its tools bundle may say what it declares** (novox/hq ADR 0357): the node's runtime carries - // the bundle and is granted the events the module emits, under the module's name, as for any module it - // carries — the bus's account is not who publishes them. So emits are refused only for a bus module - // with no bundle to say them. + // **But its tools bundle may say and ask what it declares** (novox/hq ADR 0357): the node's runtime carries + // the bundle, is granted the events the module emits under the module's name, as for any module it carries, + // and asks every tool and seat verb on the bundle's behalf (mesh-sdk go/buscompose, the runtime's invoke of + // `*`) — the bus's account is not who publishes or asks. So emits and invokes are refused only for a bus + // module with no bundle to say or ask them (issue 774: nats's bundle records its hand act). if _, account := m.OwnSecrets["broker"]; account && m.ClaimsSeat(catalogue.BrokerSeat) { var said []string - if len(m.EmitsAll()) > 0 && !manifestShipsABundle(m) { + bundled := manifestShipsABundle(m) + if len(m.EmitsAll()) > 0 && !bundled { said = append(said, "emits") } + if len(m.Invokes) > 0 && !bundled { + said = append(said, "invokes") + } for _, f := range []struct { name string n int - }{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"invokes", len(m.Invokes)}, + }{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"state", len(m.State)}, {"logs", len(m.Logs)}, {"reads", len(m.Reads)}, {"streams", len(m.Streams)}} { if f.n > 0 { diff --git a/modules/mesh-delivery-indexer/judge/rules_test.go b/modules/mesh-delivery-indexer/judge/rules_test.go index 1d6c599..75bd736 100644 --- a/modules/mesh-delivery-indexer/judge/rules_test.go +++ b/modules/mesh-delivery-indexer/judge/rules_test.go @@ -4337,6 +4337,41 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "granted the bus's snapshot API and nothing else") { t.Fatalf("a bus module declaring what it emits was not refused, or not for the reason: %v", err) } + // Nor what it invokes, with no bundle for the runtime to ask on its behalf. + unbundledInvoke := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}], + "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, + "invokes":["seat:mesh-hand-acts.hand-act"]}`) + if _, err := ParseManifest(unbundledInvoke); err == nil || !strings.Contains(err.Error(), "its invokes would be granted nothing") { + t.Fatalf("a bus module with no bundle declaring what it invokes was not refused, or not for the reason: %v", err) + } + + // Its tools bundle asks what the module invokes (novox/hq ADR 0357; issue 774): the bundle asks through the + // node's runtime, which is granted every tool and seat verb (mesh-sdk go/buscompose), so a bus module that + // ships one may declare them. The shape of mesh-catalog's modules/nats/module.json at fc5b849d, cut to the + // parts the rule reads, was refused and held every bus user list from 2026-11-02 01:35. + nats := []byte(`{"module":"nats","version":"1","provides":[{"name":"mesh-bus","scope":"mesh","credential":{"form":"none"}}], + "claims":[{"name":"mesh-broker","scope":"mesh"}], + "own-secrets":{"broker":"/run/broker","broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"}, + "resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/nats-tools","mode":"0711","unknown":"program"}], + "emits":["consumer-removed","bucket-keys-removed","stream-removed","bucket-expiry-set"], + "invokes":["seat:mesh-hand-acts.hand-act"],"consumes":[], + "tools":["nats_server","nats_bucket_expiry"], + "build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/nats-tools", + "binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["DEAD_LETTER_NOTICES.controller","ASSIGNMENTS"]}}]}}`) + nm, err := ParseManifest(nats) + if err != nil { + t.Fatalf("the bus module whose tools bundle invokes a seat verb was refused: %v", err) + } + nbuilt, err := nm.Resolve([]catalogue.Built{{Name: "tools", Kind: catalogue.ArtifactBundle, Reference: "store/tools:1", Digest: "sha256:1"}}) + if err != nil { + t.Fatal(err) + } + if raw, err = json.Marshal(nbuilt); err != nil { + t.Fatal(err) + } + if _, err := ParseManifestForInputs(raw); err != nil { + t.Fatalf("the built manifest of the bus module whose tools bundle invokes a seat verb was refused: %v", err) + } // Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is // granted the module's events, so a bus module that ships one may declare them.