mesh-enrolment: catch up only on an ask answered yes, and drop the address key first at confirm, decline and refused add

ADR 0589 version 29, clause 6. At start and after a catch-up an address key is an expiry never heard only when its enrolment key is gone and the router records its ask answered yes; one declined or not answered yet is no expiry, and one not answered stays until its ask ends. A confirm, a decline and a refused add delete the address key before their other steps, so one stopped midway never leaves a key the catch-up would take for an expiry and unplace a node that joined. A refused add (mesh-placement's refusal of the proposal, not a call that did not reach it) now ends the enrolment as refused; a call that failed still waits for again. Tests: rows in the end table (after a confirm stopped before its enrolment key went, after a decline, after a refused add, a declined ask, an ask not answered: nothing unplaced) and the integration test's holder-was-down case reading the yes; no new function. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 01:45:53 +01:00
parent 44fee9eb0e
commit 795724f5ca
6 changed files with 143 additions and 0 deletions
@@ -32,6 +32,11 @@ func (h *Holder) Confirm(caller, node string) (map[string]any, error) {
 	if e.Stage != StageRedeeming {
 		return nil, fmt.Errorf("the enrolment of %s is %s: it is confirmed after redeem", own, e.Stage)
 	}
+	// The address key goes first (ADR 0589 §6): a confirm stopped midway never leaves one the catch-up would take for an
+	// expiry and unplace a node that joined.
+	if err := h.dropAddress(e); err != nil {
+		return nil, fmt.Errorf("%w (call confirm again)", err)
+	}
 	if err := h.sayEvent(EventEnrolled, e, ""); err != nil {
 		return nil, fmt.Errorf("node-enrolled of %s cannot be said, so nothing is deleted (call confirm again): %w", own, err)
 	}
@@ -131,6 +131,14 @@ func (h *Holder) reconcileLocked() error {
 		if err != nil || r.Ask == "" || open[r.Node] {
 			continue
 		}
+		// Only an ask answered yes made a token and placed the node (ADR 0589 §6): one declined, or not answered yet, is no
+		// expiry, and one not answered stays until its ask ends.
+		if yes, err := h.answeredYes(r.Ask); err != nil {
+			errs = append(errs, fmt.Errorf("the answer to %s cannot be read, so the address %s waits for the next pass: %w", r.Ask, address, err))
+			continue
+		} else if !yes {
+			continue
+		}
 		h.logf("the enrolment of %s (ask %s) is gone and its address %s is still reserved: it expired unseen, and is ended now", r.Node, r.Ask, address)
 		if err := h.endExpired(Enrolment{Node: r.Node, Address: address, Ask: r.Ask, Issued: r.Issued}); err != nil {
 			errs = append(errs, err)
@@ -139,6 +147,15 @@ func (h *Holder) reconcileLocked() error {
 	return errors.Join(errs...)
 }

+// answeredYes reads the router's record of ask: whether the operator answered it yes.
+func (h *Holder) answeredYes(ask string) (bool, error) {
+	_, w, err := h.whereIs(ask)
+	if err != nil {
+		return false, err
+	}
+	return w != nil && w.Option == "yes", nil
+}
+
 // endExpired ends an enrolment whose key expired, in this order, each step safe to do again: the node, placed on the yes and
 // never confirmed, is taken out of placement on the warrant of its ask (only when placement holds it); enrolment-expired names
 // its issued event (said under one id, so said again is said once); the vault drops the token; the address key goes last, so
@@ -2,6 +2,8 @@ package main

 import (
 	"encoding/json"
+	"errors"
+	"git.novox.be/novox/mesh-sdk/go/asks"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"reflect"
 	"slices"
@@ -72,6 +74,7 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) {
 		{"the holder was down, and its drop failed once", "down", "drop down", func(r *rig) error { return r.h.Reconcile() }},
 	} {
 		r.seed(c.node, "10.9.0.9", StageLive)
+		r.asker.answer("ask-"+c.node, "yes")
 		issued := r.now.Add(-time.Minute)
 		_ = r.st.Delete(NodeKey(c.node)) // the bus took the key; the watch never handed its content over
 		if c.fail != "" {
@@ -92,6 +95,70 @@ func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) {
 			t.Errorf("%s: said %s (%v), placed %v, last placement call %+v, vault %v, keys %v", c.name, last.Body, err, placed, removed, r.vault.calls, r.keys())
 		}
 	}
+	// ADR 0589 §6, the negative rows: an address key the catch-up finds with its enrolment key gone is no expiry when its ask
+	// was declined or is not answered yet, and none is left behind by a confirm, a decline or a refused add, which delete it
+	// first. In none of them is the node unplaced.
+	for _, c := range []struct {
+		name string
+		end  func(r *rig, node string)
+		key  bool // the address key stays (an ask not answered yet stays until its ask ends)
+	}{
+		{"after a confirm stopped before the enrolment key went", func(r *rig, node string) {
+			e, _ := r.enrolment(node)
+			e.Stage, e.KeysDigest = StageRedeeming, delivery.KeysDigest("pk", "ov", "se", "sv")
+			_, _ = r.st.Put(NodeKey(node), e, 0)
+			r.asker.answer("ask-"+node, "yes")
+			r.st.failDelete[NodeKey(node)] = true // the confirm stops after the address key, before the enrolment key
+			_, _ = r.h.Confirm(NodeUser(node), "")
+			_ = r.st.Delete(NodeKey(node)) // and the key is gone by the next pass
+		}, false},
+		{"after a decline", func(r *rig, node string) {
+			_ = r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: "ask-" + node})
+			r.asker.answer("ask-"+node, "no")
+		}, false},
+		{"after a refused add", func(r *rig, node string) {
+			delete(r.rd["mesh-delivery-assigner.placement"], node+".node") // the add is tried, and refused
+			r.placeErr = errors.New("mesh-placement did not take the node: the proposal is refused: change 1: an address taken")
+			_ = r.h.Warranted(yes, asks.Warrant{Ask: "ask-" + node})
+			r.asker.answer("ask-"+node, "yes")
+			r.placeErr = nil
+			if !slices.Contains(r.events(), EventRefused) {
+				r.t.Errorf("a refused add did not end the enrolment: %v", r.events())
+			}
+		}, false},
+		{"a declined ask whose enrolment key is gone", func(r *rig, node string) {
+			r.asker.answer("ask-"+node, "no")
+			_ = r.st.Delete(NodeKey(node))
+		}, true},
+		{"an ask not answered yet", func(r *rig, node string) { _ = r.st.Delete(NodeKey(node)) }, true},
+	} {
+		r := newRig(t)
+		stage := StageAsked
+		if strings.HasPrefix(c.name, "after a confirm") {
+			stage = StageLive
+		}
+		r.seed("joiner", "10.9.0.11", stage)
+		r.rd["mesh-delivery-assigner.placement"]["joiner.node"] = `{"v":1,"node":"joiner","decided":{"address":"10.9.0.11"},"modules":[]}`
+		c.end(r, "joiner")
+		removes := 0
+		for _, a := range r.added {
+			if a.Change.Op == delivery.OpRemoveNode {
+				removes++
+			}
+		}
+		if err := r.h.Reconcile(); err != nil {
+			t.Errorf("%s: %v", c.name, err)
+		}
+		after := 0
+		for _, a := range r.added {
+			if a.Change.Op == delivery.OpRemoveNode {
+				after++
+			}
+		}
+		if after != removes || r.has(AddressKey("10.9.0.11")) != c.key || slices.Contains(r.events(), EventExpired) {
+			t.Errorf("%s: the catch-up unplaced (%d to %d), address key %v, events %v", c.name, removes, after, r.has(AddressKey("10.9.0.11")), r.events())
+		}
+	}
 	// An address reserved by an issue under way (no ask yet) is never taken for an expired enrolment.
 	_, _ = r.st.Put(AddressKey("10.9.0.10"), Reservation{Node: "busy"}, 0)
 	if err := r.h.Reconcile(); err != nil || !r.has(AddressKey("10.9.0.10")) {
@@ -425,6 +425,7 @@ func TestMeshEnrolmentWholeOnARealBus(t *testing.T) {
 	if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: b.Ask}); err != nil {
 		t.Fatal(err)
 	}
+	asker.answer(b.Ask, "yes") // the router's record of the answer, which the catch-up reads (ADR 0589 §6)
 	if _, err := placement.Get(ctx, "novox.node"); err != nil {
 		t.Fatalf("novox was not placed on the yes: %v", err)
 	}
@@ -260,6 +260,10 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error {
 	// The node enters placement now, with its tunnel key, on this warrant: the hub peers it before the machine redeems over
 	// the tunnel (the operator's decision of 2026-11-02 01:05). Only then is the token made.
 	if err := h.place(e, placements); err != nil {
+		if addRefused(err) {
+			// mesh-placement refused the node itself, not the call to it: the enrolment ends (ADR 0589 §6).
+			return h.refuse(e, "mesh-placement refused the node: "+err.Error())
+		}
 		return fmt.Errorf("the node %s is not added to placement, so no token is made (mesh-enrolment.again asks again): %w", e.Node, err)
 	}
 	if _, err := h.Vault.Token(e.Node, e.Expires, tunnel); err != nil {
@@ -273,8 +277,37 @@ func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error {
 	return h.sayEvent(EventIssued, e, "")
 }

+// addRefused says mesh-placement answered the add-node with a refusal of the proposal (the assigner's words, "the proposal is
+// refused: …"), as against a call that did not reach it or was not answered, which `again` asks once more.
+func addRefused(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "the proposal is refused")
+}
+
+// dropAddress deletes the enrolment's address key while it still names this node: the first step of a confirm, a decline and a
+// refused add (ADR 0589 §6), so the catch-up never takes a key left by one of them for an expiry never heard.
+func (h *Holder) dropAddress(e Enrolment) error {
+	row, found, err := h.Store.Get(AddressKey(e.Address))
+	if err != nil {
+		return fmt.Errorf("the address key %s cannot be read, so the end of %s waits: %w", e.Address, e.Node, err)
+	}
+	if !found {
+		return nil
+	}
+	if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node != e.Node {
+		return nil // another enrolment holds the address now
+	}
+	if err := h.Store.Delete(AddressKey(e.Address)); err != nil {
+		return fmt.Errorf("the address key %s is not deleted, so the end of %s waits: %w", e.Address, e.Node, err)
+	}
+	return nil
+}
+
 // refuse ends an enrolment at stage asked: both keys go and enrolment-refused carries the key's content (ADR 0587 §4).
 func (h *Holder) refuse(e Enrolment, reason string) error {
+	// The address key goes first (ADR 0589 §6): a refusal stopped midway never leaves one that looks like an expiry.
+	if err := h.dropAddress(e); err != nil {
+		return err
+	}
 	// A yes whose token failed added the node already; a refusal takes it off again.
 	if err := h.unplace(e, reason); err != nil {
 		return fmt.Errorf("%s is not taken off placement, so its enrolment stays: %w", e.Node, err)
@@ -158,6 +158,26 @@ type fakeAsker struct {
 	mu               sync.Mutex
 	asked, cancelled []string
 	err              error
+	answers          map[string]string // the router's record: the option each ask was answered with
+}
+
+// answer records that the operator answered ask with option, as the router keeps it.
+func (a *fakeAsker) answer(ask, option string) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if a.answers == nil {
+		a.answers = map[string]string{}
+	}
+	a.answers[ask] = option
+}
+
+func (a *fakeAsker) Where(id string) (string, *asks.Warrant, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if o, ok := a.answers[id]; ok {
+		return "answered", &asks.Warrant{Ask: id, Option: o}, nil
+	}
+	return "", nil, nil // no record read: as a router whose record of the ask is gone
 }

 func (a *fakeAsker) Ask(k asks.Ask) error {