From 795724f5ca6cfd8d9b3e2767733a71aa09796dad Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 01:30:04 +0100 Subject: [PATCH] mesh-enrolment: catch up only on an ask answered yes, and drop the address key first at confirm, decline and refused add ADR 0589 version 29, clause 6. At start and after a catch-up an address key is an expiry never heard only when its enrolment key is gone and the router records its ask answered yes; one declined or not answered yet is no expiry, and one not answered stays until its ask ends. A confirm, a decline and a refused add delete the address key before their other steps, so one stopped midway never leaves a key the catch-up would take for an expiry and unplace a node that joined. A refused add (mesh-placement's refusal of the proposal, not a call that did not reach it) now ends the enrolment as refused; a call that failed still waits for again. Tests: rows in the end table (after a confirm stopped before its enrolment key went, after a decline, after a refused add, a declined ask, an ask not answered: nothing unplaced) and the integration test's holder-was-down case reading the yes; no new function. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-enrolment/confirm.go | 5 ++ .../mesh-enrolment/cmd/mesh-enrolment/end.go | 17 +++++ .../cmd/mesh-enrolment/end_test.go | 67 +++++++++++++++++++ .../cmd/mesh-enrolment/integration_test.go | 1 + .../cmd/mesh-enrolment/issue.go | 33 +++++++++ .../cmd/mesh-enrolment/kit_test.go | 20 ++++++ 6 files changed, 143 insertions(+) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go index ee8977a..9324bcd 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go @@ -32,6 +32,11 @@ func (h *Holder) Confirm(caller, node string) (map[string]any, error) { if e.Stage != StageRedeeming { return nil, fmt.Errorf("the enrolment of %s is %s: it is confirmed after redeem", own, e.Stage) } + // The address key goes first (ADR 0589 §6): a confirm stopped midway never leaves one the catch-up would take for an + // expiry and unplace a node that joined. + if err := h.dropAddress(e); err != nil { + return nil, fmt.Errorf("%w (call confirm again)", err) + } if err := h.sayEvent(EventEnrolled, e, ""); err != nil { return nil, fmt.Errorf("node-enrolled of %s cannot be said, so nothing is deleted (call confirm again): %w", own, err) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go index 1a00b8b..47f045a 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -131,6 +131,14 @@ func (h *Holder) reconcileLocked() error { if err != nil || r.Ask == "" || open[r.Node] { continue } + // Only an ask answered yes made a token and placed the node (ADR 0589 §6): one declined, or not answered yet, is no + // expiry, and one not answered stays until its ask ends. + if yes, err := h.answeredYes(r.Ask); err != nil { + errs = append(errs, fmt.Errorf("the answer to %s cannot be read, so the address %s waits for the next pass: %w", r.Ask, address, err)) + continue + } else if !yes { + continue + } h.logf("the enrolment of %s (ask %s) is gone and its address %s is still reserved: it expired unseen, and is ended now", r.Node, r.Ask, address) if err := h.endExpired(Enrolment{Node: r.Node, Address: address, Ask: r.Ask, Issued: r.Issued}); err != nil { errs = append(errs, err) @@ -139,6 +147,15 @@ func (h *Holder) reconcileLocked() error { return errors.Join(errs...) } +// answeredYes reads the router's record of ask: whether the operator answered it yes. +func (h *Holder) answeredYes(ask string) (bool, error) { + _, w, err := h.whereIs(ask) + if err != nil { + return false, err + } + return w != nil && w.Option == "yes", nil +} + // endExpired ends an enrolment whose key expired, in this order, each step safe to do again: the node, placed on the yes and // never confirmed, is taken out of placement on the warrant of its ask (only when placement holds it); enrolment-expired names // its issued event (said under one id, so said again is said once); the vault drops the token; the address key goes last, so diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go index 40f1d71..a341180 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -2,6 +2,8 @@ package main import ( "encoding/json" + "errors" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" "reflect" "slices" @@ -72,6 +74,7 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { {"the holder was down, and its drop failed once", "down", "drop down", func(r *rig) error { return r.h.Reconcile() }}, } { r.seed(c.node, "10.9.0.9", StageLive) + r.asker.answer("ask-"+c.node, "yes") issued := r.now.Add(-time.Minute) _ = r.st.Delete(NodeKey(c.node)) // the bus took the key; the watch never handed its content over if c.fail != "" { @@ -92,6 +95,70 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { t.Errorf("%s: said %s (%v), placed %v, last placement call %+v, vault %v, keys %v", c.name, last.Body, err, placed, removed, r.vault.calls, r.keys()) } } + // ADR 0589 §6, the negative rows: an address key the catch-up finds with its enrolment key gone is no expiry when its ask + // was declined or is not answered yet, and none is left behind by a confirm, a decline or a refused add, which delete it + // first. In none of them is the node unplaced. + for _, c := range []struct { + name string + end func(r *rig, node string) + key bool // the address key stays (an ask not answered yet stays until its ask ends) + }{ + {"after a confirm stopped before the enrolment key went", func(r *rig, node string) { + e, _ := r.enrolment(node) + e.Stage, e.KeysDigest = StageRedeeming, delivery.KeysDigest("pk", "ov", "se", "sv") + _, _ = r.st.Put(NodeKey(node), e, 0) + r.asker.answer("ask-"+node, "yes") + r.st.failDelete[NodeKey(node)] = true // the confirm stops after the address key, before the enrolment key + _, _ = r.h.Confirm(NodeUser(node), "") + _ = r.st.Delete(NodeKey(node)) // and the key is gone by the next pass + }, false}, + {"after a decline", func(r *rig, node string) { + _ = r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: "ask-" + node}) + r.asker.answer("ask-"+node, "no") + }, false}, + {"after a refused add", func(r *rig, node string) { + delete(r.rd["mesh-delivery-assigner.placement"], node+".node") // the add is tried, and refused + r.placeErr = errors.New("mesh-placement did not take the node: the proposal is refused: change 1: an address taken") + _ = r.h.Warranted(yes, asks.Warrant{Ask: "ask-" + node}) + r.asker.answer("ask-"+node, "yes") + r.placeErr = nil + if !slices.Contains(r.events(), EventRefused) { + r.t.Errorf("a refused add did not end the enrolment: %v", r.events()) + } + }, false}, + {"a declined ask whose enrolment key is gone", func(r *rig, node string) { + r.asker.answer("ask-"+node, "no") + _ = r.st.Delete(NodeKey(node)) + }, true}, + {"an ask not answered yet", func(r *rig, node string) { _ = r.st.Delete(NodeKey(node)) }, true}, + } { + r := newRig(t) + stage := StageAsked + if strings.HasPrefix(c.name, "after a confirm") { + stage = StageLive + } + r.seed("joiner", "10.9.0.11", stage) + r.rd["mesh-delivery-assigner.placement"]["joiner.node"] = `{"v":1,"node":"joiner","decided":{"address":"10.9.0.11"},"modules":[]}` + c.end(r, "joiner") + removes := 0 + for _, a := range r.added { + if a.Change.Op == delivery.OpRemoveNode { + removes++ + } + } + if err := r.h.Reconcile(); err != nil { + t.Errorf("%s: %v", c.name, err) + } + after := 0 + for _, a := range r.added { + if a.Change.Op == delivery.OpRemoveNode { + after++ + } + } + if after != removes || r.has(AddressKey("10.9.0.11")) != c.key || slices.Contains(r.events(), EventExpired) { + t.Errorf("%s: the catch-up unplaced (%d to %d), address key %v, events %v", c.name, removes, after, r.has(AddressKey("10.9.0.11")), r.events()) + } + } // An address reserved by an issue under way (no ask yet) is never taken for an expired enrolment. _, _ = r.st.Put(AddressKey("10.9.0.10"), Reservation{Node: "busy"}, 0) if err := r.h.Reconcile(); err != nil || !r.has(AddressKey("10.9.0.10")) { diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go index b351880..fe757eb 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -425,6 +425,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: b.Ask}); err != nil { t.Fatal(err) } + asker.answer(b.Ask, "yes") // the router's record of the answer, which the catch-up reads (ADR 0589 §6) if _, err := placement.Get(ctx, "novox.node"); err != nil { t.Fatalf("novox was not placed on the yes: %v", err) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go index e52f3e8..597f542 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -260,6 +260,10 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { // The node enters placement now, with its tunnel key, on this warrant: the hub peers it before the machine redeems over // the tunnel (the operator's decision of 2026-11-02 01:05). Only then is the token made. if err := h.place(e, placements); err != nil { + if addRefused(err) { + // mesh-placement refused the node itself, not the call to it: the enrolment ends (ADR 0589 §6). + return h.refuse(e, "mesh-placement refused the node: "+err.Error()) + } return fmt.Errorf("the node %s is not added to placement, so no token is made (mesh-enrolment.again asks again): %w", e.Node, err) } if _, err := h.Vault.Token(e.Node, e.Expires, tunnel); err != nil { @@ -273,8 +277,37 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { return h.sayEvent(EventIssued, e, "") } +// addRefused says mesh-placement answered the add-node with a refusal of the proposal (the assigner's words, "the proposal is +// refused: …"), as against a call that did not reach it or was not answered, which `again` asks once more. +func addRefused(err error) bool { + return err != nil && strings.Contains(err.Error(), "the proposal is refused") +} + +// dropAddress deletes the enrolment's address key while it still names this node: the first step of a confirm, a decline and a +// refused add (ADR 0589 §6), so the catch-up never takes a key left by one of them for an expiry never heard. +func (h *Holder) dropAddress(e Enrolment) error { + row, found, err := h.Store.Get(AddressKey(e.Address)) + if err != nil { + return fmt.Errorf("the address key %s cannot be read, so the end of %s waits: %w", e.Address, e.Node, err) + } + if !found { + return nil + } + if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node != e.Node { + return nil // another enrolment holds the address now + } + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s is not deleted, so the end of %s waits: %w", e.Address, e.Node, err) + } + return nil +} + // refuse ends an enrolment at stage asked: both keys go and enrolment-refused carries the key's content (ADR 0587 §4). func (h *Holder) refuse(e Enrolment, reason string) error { + // The address key goes first (ADR 0589 §6): a refusal stopped midway never leaves one that looks like an expiry. + if err := h.dropAddress(e); err != nil { + return err + } // A yes whose token failed added the node already; a refusal takes it off again. if err := h.unplace(e, reason); err != nil { return fmt.Errorf("%s is not taken off placement, so its enrolment stays: %w", e.Node, err) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go index 45a17b6..4dbad2b 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -158,6 +158,26 @@ type fakeAsker struct { mu sync.Mutex asked, cancelled []string err error + answers map[string]string // the router's record: the option each ask was answered with +} + +// answer records that the operator answered ask with option, as the router keeps it. +func (a *fakeAsker) answer(ask, option string) { + a.mu.Lock() + defer a.mu.Unlock() + if a.answers == nil { + a.answers = map[string]string{} + } + a.answers[ask] = option +} + +func (a *fakeAsker) Where(id string) (string, *asks.Warrant, error) { + a.mu.Lock() + defer a.mu.Unlock() + if o, ok := a.answers[id]; ok { + return "answered", &asks.Warrant{Ask: id, Option: o}, nil + } + return "", nil, nil // no record read: as a router whose record of the ask is gone } func (a *fakeAsker) Ask(k asks.Ask) error {