diff --git a/cmd/mesh-host/enrol_seat.go b/cmd/mesh-host/enrol_seat.go index 55a71c3..4d253a8 100644 --- a/cmd/mesh-host/enrol_seat.go +++ b/cmd/mesh-host/enrol_seat.go @@ -63,6 +63,10 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n answer, err := j.Run(ctx) switch { case errors.Is(err, link.ErrNoSeat): + // The path before the lease write: the controller's enrolment subject is granted to the token only while the + // controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said. + fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+ + "subject, the path before the lease write, which the bus refuses once the lease is written\n", err) return false, nil case err != nil: return true, err diff --git a/cmd/mesh-host/logins.go b/cmd/mesh-host/logins.go index f7e923c..ab29a6f 100644 --- a/cmd/mesh-host/logins.go +++ b/cmd/mesh-host/logins.go @@ -131,7 +131,8 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str func queueNode(q *link.Session) string { return q.Membership.Node } // newRotator is the engine's side of a rotation over this session (ADR 0583 §3-§6): the value fetched through the vault's own -// hand-out, the test-dial on a second connection, the confirmation over the new link, and the finding said in the journal. +// hand-out, the test-dial on a second connection, the confirmation over the new link, and a fall back carried in the health +// report. func newRotator(queue *link.Session, statePath string, timeout time.Duration, say link.Announce) *link.Rotator { r := &link.Rotator{Logins: queue.Membership.Logins, Say: say, Fetch: fetchLogin(queue, statePath), @@ -145,9 +146,9 @@ func newRotator(queue *link.Session, statePath string, timeout time.Duration, sa } _, err := queue.AskVault(ctx, "node-account-confirm", map[string]any{}) return err - }, - // The finding the watchman raises (ADR 0583 §6): said in the journal, the route an engine says a finding by. - Finding: func(name, text string) { say("finding " + name + ": " + text) }} + }} r.Watch() + // The fall back is carried in this node's health report, from which the watchman raises node-account-fallback (ADR 0583 §6). + queue.Fallback = r.Fallback return r } diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go index b938dbb..5aa2d87 100644 --- a/internal/link/nodehealth.go +++ b/internal/link/nodehealth.go @@ -26,7 +26,8 @@ import ( // read across every node's report; // - machine-units: Units, the failed system units no module owns; // - relogin-needed: Relogin, each account whose running session lacks a group it was given; -// - directory-used-as-found: Found, each directory a module declares that the engine uses as found. +// - directory-used-as-found: Found, each directory a module declares that the engine uses as found; +// - node-account-fallback: Fallback, the engine's fall back to its old login during a rotation (ADR 0583 §6). // // The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts. @@ -176,6 +177,9 @@ func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, erro if err != nil { return false, err } + if q.Fallback != nil && h.Fallback == nil { + h.Fallback = q.Fallback() // node-account-fallback (ADR 0583 §6) + } return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now) } diff --git a/internal/link/rotate.go b/internal/link/rotate.go index d5c8f02..12d06bb 100644 --- a/internal/link/rotate.go +++ b/internal/link/rotate.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -22,9 +23,10 @@ type Rotator struct { TestDial func(ctx context.Context, login, password string) error // Confirm calls mesh-vault.node-account-confirm over the link as the login now in use, once it is open. Confirm func(ctx context.Context) error - // Say is the engine's journal; Finding says a finding the watchman raises (node-account-fallback). - Say func(string) - Finding func(name, text string) + // Say is the engine's journal. + Say func(string) + // Now is the clock a fall back is dated by; time.Now when nil. + Now func() time.Time // FailBound is how many authentication failures with the new login, before rotation-done, it falls back after; 3 when 0. FailBound int @@ -33,6 +35,9 @@ type Rotator struct { failures int // fellBack is the key's `since` a fallback was made for: the same rotation is not tried again. fellBackSince string + // fallen is the fall back the node's health report carries (delivery.LoginFallback), from which the watchman raises + // node-account-fallback (ADR 0583 §6), until the vault's key of this node says anything new. + fallen *delivery.LoginFallback switchedSince string } @@ -46,6 +51,9 @@ func (r *Rotator) say(f string, a ...any) { func (r *Rotator) Heard(ctx context.Context, nl delivery.NodeLogin) { r.mu.Lock() defer r.mu.Unlock() + if r.fallen != nil && nl.Since.String() != r.fellBackSince { + r.fallen = nil // the vault moved on (abandoned, done, or another rotation begun): the fall back is no longer said + } cur, _ := r.Logins.Current() switch { case nl.Rotating == "" && nl.Login == cur: @@ -149,14 +157,30 @@ func (r *Rotator) AuthFailed(login string) { return } text := fmt.Sprintf("%s failed to authenticate %d times before the vault said rotation-done: this node dials as %s again", login, r.failures, r.old) - r.say("%s", text) - if r.Finding != nil { - r.Finding(delivery.NodeAccountFallbackFinding, text) + r.say("%s: %s", delivery.NodeAccountFallbackFinding, text) + now := time.Now + if r.Now != nil { + now = r.Now } + r.fallen = &delivery.LoginFallback{Node: r.Logins.node, From: login, Why: text, At: now().UTC()} r.old, r.failures = "", 0 r.fellBackSince = r.switchedSince } +// Fallback is the fall back the node's health report carries now, nil when there is none. +func (r *Rotator) Fallback() *delivery.LoginFallback { + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + if r.fallen == nil { + return nil + } + f := *r.fallen + return &f +} + // AuthOK is told of each successful connection as login: the failures with it are forgotten. func (r *Rotator) AuthOK(login string) { r.mu.Lock() diff --git a/internal/link/rotate_test.go b/internal/link/rotate_test.go index 89388e9..57a8e32 100644 --- a/internal/link/rotate_test.go +++ b/internal/link/rotate_test.go @@ -19,7 +19,6 @@ type rotRig struct { steps []string dialErr error said []string - finding string } func newRotRig() *rotRig { @@ -46,7 +45,7 @@ func newRotRig() *rotRig { return nil }, Say: func(s string) { g.said = append(g.said, s) }, - Finding: func(name, text string) { g.finding = name }, + Now: func() time.Time { return time.Date(2026, 11, 2, 12, 5, 0, 0, time.UTC) }, FailBound: 2} g.r.Watch() return g @@ -136,8 +135,13 @@ func TestTheOldValueGoesAtRotationDoneAbandonSwitchesBackAndFailuresFallBack(t * t.Fatalf("fell back after one failure: %s", cur) } f.logins.dialed("node.ace--b", errors.New("nats: authorization violation")) - if cur, _ := f.logins.Current(); cur != "node.ace" || f.finding != delivery.NodeAccountFallbackFinding { - t.Fatalf("after the bound: %s, finding %q", cur, f.finding) + fb := f.r.Fallback() + if cur, _ := f.logins.Current(); cur != "node.ace" || fb == nil || fb.Node != "ace" || fb.From != "node.ace--b" || fb.Why == "" || fb.At.IsZero() { + t.Fatalf("after the bound: %s, fall back %+v", cur, fb) + } + // The health report carries it, from which the watchman raises node-account-fallback. + if _, err := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: "ace", Fallback: fb}); err != nil { + t.Errorf("the fall back is no health report's: %v", err) } steps := len(f.steps) f.r.Heard(ctx, key("node.ace", "node.ace--b", delivery.RotationReady)) @@ -145,4 +149,11 @@ func TestTheOldValueGoesAtRotationDoneAbandonSwitchesBackAndFailuresFallBack(t * t.Errorf("the rotation fallen back from was tried again: %v", f.steps[steps:]) } f.logins.dialed("node.ace", errors.New("dial tcp: connection refused")) // not a refusal of credentials: counts for nothing + // The vault moves on (it abandons the rotation): the fall back is said no more. + abandoned := key("node.ace", "", "") + abandoned.Since = abandoned.Since.Add(time.Minute) + f.r.Heard(ctx, abandoned) + if f.r.Fallback() != nil { + t.Error("the fall back is still said after the vault moved on") + } } diff --git a/internal/link/session.go b/internal/link/session.go index dfe1d3b..832348e 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery" "sync" "time" @@ -24,6 +25,8 @@ const Writer = "node-engine" // event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles // a whole declaration. type Session struct { + // Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none. + Fallback func() *delivery.LoginFallback // linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs). linkedUser string // Membership is the node and the mesh's key a controller's ask is verified with.