The engine carries its login fall back in its health report, and says the enrolment fallback is the pre-lease path

A fall back to the old login during a rotation is now delivery.NodeHealth's Fallback (the node, the login fallen back from,
why, when), written with the node's health report, from which the watchman raises node-account-fallback; it is said no more
once the vault's key of the node moves on (blocker 7, ADR 0583 §6). mesh-host enrol's fall back to the controller's
enrolment subject says it is the path before the lease write, which the bus refuses once the lease is written. Tests: rows
in the existing rotation test; none added.

(issue 774)

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-02 01:49:01 +01:00
parent 72aca552e1
commit 12168ba4da
6 changed files with 62 additions and 15 deletions
+4
View File
@@ -63,6 +63,10 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n
answer, err := j.Run(ctx)
switch {
case errors.Is(err, link.ErrNoSeat):
// The path before the lease write: the controller's enrolment subject is granted to the token only while the
// controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said.
fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+
"subject, the path before the lease write, which the bus refuses once the lease is written\n", err)
return false, nil
case err != nil:
return true, err
+5 -4
View File
@@ -131,7 +131,8 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str
func queueNode(q *link.Session) string { return q.Membership.Node }
// newRotator is the engine's side of a rotation over this session (ADR 0583 §3-§6): the value fetched through the vault's own
// hand-out, the test-dial on a second connection, the confirmation over the new link, and the finding said in the journal.
// hand-out, the test-dial on a second connection, the confirmation over the new link, and a fall back carried in the health
// report.
func newRotator(queue *link.Session, statePath string, timeout time.Duration, say link.Announce) *link.Rotator {
r := &link.Rotator{Logins: queue.Membership.Logins, Say: say,
Fetch: fetchLogin(queue, statePath),
@@ -145,9 +146,9 @@ func newRotator(queue *link.Session, statePath string, timeout time.Duration, sa
}
_, err := queue.AskVault(ctx, "node-account-confirm", map[string]any{})
return err
},
// The finding the watchman raises (ADR 0583 §6): said in the journal, the route an engine says a finding by.
Finding: func(name, text string) { say("finding " + name + ": " + text) }}
}}
r.Watch()
// The fall back is carried in this node's health report, from which the watchman raises node-account-fallback (ADR 0583 §6).
queue.Fallback = r.Fallback
return r
}
+5 -1
View File
@@ -26,7 +26,8 @@ import (
// read across every node's report;
// - machine-units: Units, the failed system units no module owns;
// - relogin-needed: Relogin, each account whose running session lacks a group it was given;
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found.
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found;
// - node-account-fallback: Fallback, the engine's fall back to its old login during a rotation (ADR 0583 §6).
//
// The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts.
@@ -176,6 +177,9 @@ func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, erro
if err != nil {
return false, err
}
if q.Fallback != nil && h.Fallback == nil {
h.Fallback = q.Fallback() // node-account-fallback (ADR 0583 §6)
}
return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now)
}
+30 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -22,9 +23,10 @@ type Rotator struct {
TestDial func(ctx context.Context, login, password string) error
// Confirm calls mesh-vault.node-account-confirm over the link as the login now in use, once it is open.
Confirm func(ctx context.Context) error
// Say is the engine's journal; Finding says a finding the watchman raises (node-account-fallback).
Say func(string)
Finding func(name, text string)
// Say is the engine's journal.
Say func(string)
// Now is the clock a fall back is dated by; time.Now when nil.
Now func() time.Time
// FailBound is how many authentication failures with the new login, before rotation-done, it falls back after; 3 when 0.
FailBound int
@@ -33,6 +35,9 @@ type Rotator struct {
failures int
// fellBack is the key's `since` a fallback was made for: the same rotation is not tried again.
fellBackSince string
// fallen is the fall back the node's health report carries (delivery.LoginFallback), from which the watchman raises
// node-account-fallback (ADR 0583 §6), until the vault's key of this node says anything new.
fallen *delivery.LoginFallback
switchedSince string
}
@@ -46,6 +51,9 @@ func (r *Rotator) say(f string, a ...any) {
func (r *Rotator) Heard(ctx context.Context, nl delivery.NodeLogin) {
r.mu.Lock()
defer r.mu.Unlock()
if r.fallen != nil && nl.Since.String() != r.fellBackSince {
r.fallen = nil // the vault moved on (abandoned, done, or another rotation begun): the fall back is no longer said
}
cur, _ := r.Logins.Current()
switch {
case nl.Rotating == "" && nl.Login == cur:
@@ -149,14 +157,30 @@ func (r *Rotator) AuthFailed(login string) {
return
}
text := fmt.Sprintf("%s failed to authenticate %d times before the vault said rotation-done: this node dials as %s again", login, r.failures, r.old)
r.say("%s", text)
if r.Finding != nil {
r.Finding(delivery.NodeAccountFallbackFinding, text)
r.say("%s: %s", delivery.NodeAccountFallbackFinding, text)
now := time.Now
if r.Now != nil {
now = r.Now
}
r.fallen = &delivery.LoginFallback{Node: r.Logins.node, From: login, Why: text, At: now().UTC()}
r.old, r.failures = "", 0
r.fellBackSince = r.switchedSince
}
// Fallback is the fall back the node's health report carries now, nil when there is none.
func (r *Rotator) Fallback() *delivery.LoginFallback {
if r == nil {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
if r.fallen == nil {
return nil
}
f := *r.fallen
return &f
}
// AuthOK is told of each successful connection as login: the failures with it are forgotten.
func (r *Rotator) AuthOK(login string) {
r.mu.Lock()
+15 -4
View File
@@ -19,7 +19,6 @@ type rotRig struct {
steps []string
dialErr error
said []string
finding string
}
func newRotRig() *rotRig {
@@ -46,7 +45,7 @@ func newRotRig() *rotRig {
return nil
},
Say: func(s string) { g.said = append(g.said, s) },
Finding: func(name, text string) { g.finding = name },
Now: func() time.Time { return time.Date(2026, 11, 2, 12, 5, 0, 0, time.UTC) },
FailBound: 2}
g.r.Watch()
return g
@@ -136,8 +135,13 @@ func TestTheOldValueGoesAtRotationDoneAbandonSwitchesBackAndFailuresFallBack(t *
t.Fatalf("fell back after one failure: %s", cur)
}
f.logins.dialed("node.ace--b", errors.New("nats: authorization violation"))
if cur, _ := f.logins.Current(); cur != "node.ace" || f.finding != delivery.NodeAccountFallbackFinding {
t.Fatalf("after the bound: %s, finding %q", cur, f.finding)
fb := f.r.Fallback()
if cur, _ := f.logins.Current(); cur != "node.ace" || fb == nil || fb.Node != "ace" || fb.From != "node.ace--b" || fb.Why == "" || fb.At.IsZero() {
t.Fatalf("after the bound: %s, fall back %+v", cur, fb)
}
// The health report carries it, from which the watchman raises node-account-fallback.
if _, err := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: "ace", Fallback: fb}); err != nil {
t.Errorf("the fall back is no health report's: %v", err)
}
steps := len(f.steps)
f.r.Heard(ctx, key("node.ace", "node.ace--b", delivery.RotationReady))
@@ -145,4 +149,11 @@ func TestTheOldValueGoesAtRotationDoneAbandonSwitchesBackAndFailuresFallBack(t *
t.Errorf("the rotation fallen back from was tried again: %v", f.steps[steps:])
}
f.logins.dialed("node.ace", errors.New("dial tcp: connection refused")) // not a refusal of credentials: counts for nothing
// The vault moves on (it abandons the rotation): the fall back is said no more.
abandoned := key("node.ace", "", "")
abandoned.Since = abandoned.Since.Add(time.Minute)
f.r.Heard(ctx, abandoned)
if f.r.Fallback() != nil {
t.Error("the fall back is still said after the vault moved on")
}
}
+3
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
"sync"
"time"
@@ -24,6 +25,8 @@ const Writer = "node-engine"
// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles
// a whole declaration.
type Session struct {
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
Fallback func() *delivery.LoginFallback
// linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs).
linkedUser string
// Membership is the node and the mesh's key a controller's ask is verified with.