The engine carries its login fall back in its health report, and says the enrolment fallback is the pre-lease path
A fall back to the old login during a rotation is now delivery.NodeHealth's Fallback (the node, the login fallen back from, why, when), written with the node's health report, from which the watchman raises node-account-fallback; it is said no more once the vault's key of the node moves on (blocker 7, ADR 0583 §6). mesh-host enrol's fall back to the controller's enrolment subject says it is the path before the lease write, which the bus refuses once the lease is written. Tests: rows in the existing rotation test; none added. (issue 774) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
@@ -63,6 +63,10 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n
|
||||
answer, err := j.Run(ctx)
|
||||
switch {
|
||||
case errors.Is(err, link.ErrNoSeat):
|
||||
// The path before the lease write: the controller's enrolment subject is granted to the token only while the
|
||||
// controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said.
|
||||
fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+
|
||||
"subject, the path before the lease write, which the bus refuses once the lease is written\n", err)
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return true, err
|
||||
|
||||
@@ -131,7 +131,8 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str
|
||||
func queueNode(q *link.Session) string { return q.Membership.Node }
|
||||
|
||||
// newRotator is the engine's side of a rotation over this session (ADR 0583 §3-§6): the value fetched through the vault's own
|
||||
// hand-out, the test-dial on a second connection, the confirmation over the new link, and the finding said in the journal.
|
||||
// hand-out, the test-dial on a second connection, the confirmation over the new link, and a fall back carried in the health
|
||||
// report.
|
||||
func newRotator(queue *link.Session, statePath string, timeout time.Duration, say link.Announce) *link.Rotator {
|
||||
r := &link.Rotator{Logins: queue.Membership.Logins, Say: say,
|
||||
Fetch: fetchLogin(queue, statePath),
|
||||
@@ -145,9 +146,9 @@ func newRotator(queue *link.Session, statePath string, timeout time.Duration, sa
|
||||
}
|
||||
_, err := queue.AskVault(ctx, "node-account-confirm", map[string]any{})
|
||||
return err
|
||||
},
|
||||
// The finding the watchman raises (ADR 0583 §6): said in the journal, the route an engine says a finding by.
|
||||
Finding: func(name, text string) { say("finding " + name + ": " + text) }}
|
||||
}}
|
||||
r.Watch()
|
||||
// The fall back is carried in this node's health report, from which the watchman raises node-account-fallback (ADR 0583 §6).
|
||||
queue.Fallback = r.Fallback
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -26,7 +26,8 @@ import (
|
||||
// read across every node's report;
|
||||
// - machine-units: Units, the failed system units no module owns;
|
||||
// - relogin-needed: Relogin, each account whose running session lacks a group it was given;
|
||||
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found.
|
||||
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found;
|
||||
// - node-account-fallback: Fallback, the engine's fall back to its old login during a rotation (ADR 0583 §6).
|
||||
//
|
||||
|
||||
// The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts.
|
||||
@@ -176,6 +177,9 @@ func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, erro
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if q.Fallback != nil && h.Fallback == nil {
|
||||
h.Fallback = q.Fallback() // node-account-fallback (ADR 0583 §6)
|
||||
}
|
||||
return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now)
|
||||
}
|
||||
|
||||
|
||||
+30
-6
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -22,9 +23,10 @@ type Rotator struct {
|
||||
TestDial func(ctx context.Context, login, password string) error
|
||||
// Confirm calls mesh-vault.node-account-confirm over the link as the login now in use, once it is open.
|
||||
Confirm func(ctx context.Context) error
|
||||
// Say is the engine's journal; Finding says a finding the watchman raises (node-account-fallback).
|
||||
Say func(string)
|
||||
Finding func(name, text string)
|
||||
// Say is the engine's journal.
|
||||
Say func(string)
|
||||
// Now is the clock a fall back is dated by; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// FailBound is how many authentication failures with the new login, before rotation-done, it falls back after; 3 when 0.
|
||||
FailBound int
|
||||
|
||||
@@ -33,6 +35,9 @@ type Rotator struct {
|
||||
failures int
|
||||
// fellBack is the key's `since` a fallback was made for: the same rotation is not tried again.
|
||||
fellBackSince string
|
||||
// fallen is the fall back the node's health report carries (delivery.LoginFallback), from which the watchman raises
|
||||
// node-account-fallback (ADR 0583 §6), until the vault's key of this node says anything new.
|
||||
fallen *delivery.LoginFallback
|
||||
switchedSince string
|
||||
}
|
||||
|
||||
@@ -46,6 +51,9 @@ func (r *Rotator) say(f string, a ...any) {
|
||||
func (r *Rotator) Heard(ctx context.Context, nl delivery.NodeLogin) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.fallen != nil && nl.Since.String() != r.fellBackSince {
|
||||
r.fallen = nil // the vault moved on (abandoned, done, or another rotation begun): the fall back is no longer said
|
||||
}
|
||||
cur, _ := r.Logins.Current()
|
||||
switch {
|
||||
case nl.Rotating == "" && nl.Login == cur:
|
||||
@@ -149,14 +157,30 @@ func (r *Rotator) AuthFailed(login string) {
|
||||
return
|
||||
}
|
||||
text := fmt.Sprintf("%s failed to authenticate %d times before the vault said rotation-done: this node dials as %s again", login, r.failures, r.old)
|
||||
r.say("%s", text)
|
||||
if r.Finding != nil {
|
||||
r.Finding(delivery.NodeAccountFallbackFinding, text)
|
||||
r.say("%s: %s", delivery.NodeAccountFallbackFinding, text)
|
||||
now := time.Now
|
||||
if r.Now != nil {
|
||||
now = r.Now
|
||||
}
|
||||
r.fallen = &delivery.LoginFallback{Node: r.Logins.node, From: login, Why: text, At: now().UTC()}
|
||||
r.old, r.failures = "", 0
|
||||
r.fellBackSince = r.switchedSince
|
||||
}
|
||||
|
||||
// Fallback is the fall back the node's health report carries now, nil when there is none.
|
||||
func (r *Rotator) Fallback() *delivery.LoginFallback {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.fallen == nil {
|
||||
return nil
|
||||
}
|
||||
f := *r.fallen
|
||||
return &f
|
||||
}
|
||||
|
||||
// AuthOK is told of each successful connection as login: the failures with it are forgotten.
|
||||
func (r *Rotator) AuthOK(login string) {
|
||||
r.mu.Lock()
|
||||
|
||||
@@ -19,7 +19,6 @@ type rotRig struct {
|
||||
steps []string
|
||||
dialErr error
|
||||
said []string
|
||||
finding string
|
||||
}
|
||||
|
||||
func newRotRig() *rotRig {
|
||||
@@ -46,7 +45,7 @@ func newRotRig() *rotRig {
|
||||
return nil
|
||||
},
|
||||
Say: func(s string) { g.said = append(g.said, s) },
|
||||
Finding: func(name, text string) { g.finding = name },
|
||||
Now: func() time.Time { return time.Date(2026, 11, 2, 12, 5, 0, 0, time.UTC) },
|
||||
FailBound: 2}
|
||||
g.r.Watch()
|
||||
return g
|
||||
@@ -136,8 +135,13 @@ func TestTheOldValueGoesAtRotationDoneAbandonSwitchesBackAndFailuresFallBack(t *
|
||||
t.Fatalf("fell back after one failure: %s", cur)
|
||||
}
|
||||
f.logins.dialed("node.ace--b", errors.New("nats: authorization violation"))
|
||||
if cur, _ := f.logins.Current(); cur != "node.ace" || f.finding != delivery.NodeAccountFallbackFinding {
|
||||
t.Fatalf("after the bound: %s, finding %q", cur, f.finding)
|
||||
fb := f.r.Fallback()
|
||||
if cur, _ := f.logins.Current(); cur != "node.ace" || fb == nil || fb.Node != "ace" || fb.From != "node.ace--b" || fb.Why == "" || fb.At.IsZero() {
|
||||
t.Fatalf("after the bound: %s, fall back %+v", cur, fb)
|
||||
}
|
||||
// The health report carries it, from which the watchman raises node-account-fallback.
|
||||
if _, err := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: "ace", Fallback: fb}); err != nil {
|
||||
t.Errorf("the fall back is no health report's: %v", err)
|
||||
}
|
||||
steps := len(f.steps)
|
||||
f.r.Heard(ctx, key("node.ace", "node.ace--b", delivery.RotationReady))
|
||||
@@ -145,4 +149,11 @@ func TestTheOldValueGoesAtRotationDoneAbandonSwitchesBackAndFailuresFallBack(t *
|
||||
t.Errorf("the rotation fallen back from was tried again: %v", f.steps[steps:])
|
||||
}
|
||||
f.logins.dialed("node.ace", errors.New("dial tcp: connection refused")) // not a refusal of credentials: counts for nothing
|
||||
// The vault moves on (it abandons the rotation): the fall back is said no more.
|
||||
abandoned := key("node.ace", "", "")
|
||||
abandoned.Since = abandoned.Since.Add(time.Minute)
|
||||
f.r.Heard(ctx, abandoned)
|
||||
if f.r.Fallback() != nil {
|
||||
t.Error("the fall back is still said after the vault moved on")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -24,6 +25,8 @@ const Writer = "node-engine"
|
||||
// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles
|
||||
// a whole declaration.
|
||||
type Session struct {
|
||||
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
|
||||
Fallback func() *delivery.LoginFallback
|
||||
// linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs).
|
||||
linkedUser string
|
||||
// Membership is the node and the mesh's key a controller's ask is verified with.
|
||||
|
||||
Reference in New Issue
Block a user