novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Delete the push and the declaration: the send stub and node holds, link declare and its order, the sent readers, and the sweep judged by compositions (release 6, issue 1945 B8)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 17:12:35 +02:00
parent 66650e315c
commit f56458da32
57 changed files with 228 additions and 2587 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/inventory/build_lookups.go
-8
View File
@@ -24,11 +24,3 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s
}
return i.chainBuildFingerprints(ctx, c, module)
}
// short is a commit as a person reads it.
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
internal/inventory/nodes.go
-11
View File
@@ -903,17 +903,6 @@ type BusyWait struct {
// BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting).
const BusyWaiting = "waiting"
// WaitingFor is the container of a module that waits because it said busy, and whether one does: a
// resource whose id is `<module>.<resource>`.
func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) {
for _, b := range busy {
if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") {
return b, true
}
}
return BusyWait{}, false
}
// Waiting says whether any container waits because it said busy.
func (d Doing) Waiting() bool {
for _, b := range d.Busy {
internal/link/bus.go
-18
View File
@@ -26,11 +26,6 @@ type Bus interface {
// nobody is obliged to act (ADR 0041).
PublishEvent(ctx context.Context, key, source, node string, body []byte) error
// PublishDeclaration delivers one node what it should be. Addressed to that node alone: a
// declaration is not an event, and replaying yesterday's is actively harmful
// (design 29 §4, the *state* shape).
PublishDeclaration(ctx context.Context, node string, body []byte) error
// PublishSeatEvent states a fact under a role's own name, for the holder of that role. A
// module's event is addressed to the module; a role's is addressed to the role, so it keeps
// meaning when the holder changes (novox/hq ADR 0121, ADR 0129).
@@ -101,11 +96,6 @@ func SeatEventSubject(seat, event string) string {
return "mesh.seat." + seat + ".event." + event
}
// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so
// a node that was away gets exactly the current one and a replayed older one is refused by
// sequence — the wire-level answer to novox/hq issue 107.
func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" }
func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error {
id, err := eventID()
if err != nil {
@@ -192,14 +182,6 @@ func (b OverNATS) MaxPayload() int64 {
return b.Conn.MaxPayload()
}
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
if err != nil {
return fmt.Errorf("declaring to %s: %w", node, err)
}
return nil
}
// ToolSubject is where a module answers. Derived from the module and the tool, so a caller names
// what it wants rather than where it lives.
func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool }
internal/link/bus_test.go
-10
View File
@@ -12,16 +12,6 @@ import (
//
// go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus)
// The subject a declaration lands on is one node's, and nothing else's — the state shape.
func TestADeclarationIsAddressedToOneNode(t *testing.T) {
if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" {
t.Fatalf("a declaration would go to %q", got)
}
if DeclareSubject("anchor") == DeclareSubject("laptop") {
t.Fatal("two nodes share a declaration subject, so each would apply the other's")
}
}
// A module cannot emit under another's name: the subject is derived from the source, and the
// server's permissions make that subject the authority.
func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) {
internal/link/declare.go
-94
View File
@@ -1,94 +0,0 @@
package link
import (
"context"
"encoding/json"
"fmt"
"time"
)
// Signer is whatever holds the control plane's signing key.
type Signer interface {
Sign(ctx context.Context, message []byte) ([]byte, error)
}
// ActingGate is what every send passes before it is made (novox/hq to-be 45 §6): whether this process
// may act under the controller's lease. Set by the controller; nil passes every send — a test, a tool.
var ActingGate func(ctx context.Context) error
// Declare sends a node what it should be, signed.
//
// The signature is over the declaration exactly as it is published — the same bytes the node
// verifies. Anything that re-encoded between here and there would produce a signature over
// something else, and the node would refuse a declaration that was genuinely the mesh's.
//
// Published to the node's own queue, which its account alone may read.
func Declare(ctx context.Context, bus Bus, signer Signer, node string,
declaration []byte, timeout time.Duration) error {
if !json.Valid(declaration) {
return fmt.Errorf("refusing to send %s something that is not a declaration", node)
}
// **At the send, not only where it was composed**: a lease lost between the two stops this one.
if ActingGate != nil {
if err := ActingGate(ctx); err != nil {
return fmt.Errorf("%s was not sent its declaration: %w", node, err)
}
}
signature, err := signer.Sign(ctx, declaration)
if err != nil {
return fmt.Errorf("cannot sign a declaration for %s: %w", node, err)
}
body, err := Envelope(declaration, signature)
if err != nil {
return err
}
// **Measured before it is sent** (novox/hq issue 1689): the bare bus error said only "maximum payload
// exceeded", on 2026-10-21 for the control node's declaration, 2 KB over. The limit is the connection's own.
if limit := payloadLimitOf(bus); limit > 0 {
size := int64(len(body))
if DeclarationMeasured != nil {
DeclarationMeasured(ctx, node, size, limit)
}
if size > limit {
return fmt.Errorf("%s's declaration is %d bytes on the wire (%d bytes of declaration, signed and encoded), "+
"over the bus's largest message of %d bytes by %d: it cannot be sent (novox/hq issue 1689)",
node, size, len(declaration), limit, size-limit)
}
}
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
return bus.PublishDeclaration(publish, node, body)
}
// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON
// document. Its length is what the bus's largest message bounds.
func Envelope(declaration, signature []byte) ([]byte, error) {
return json.Marshal(Signed{Declaration: declaration, Signature: signature})
}
// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that
// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486
// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline).
const NearTheLimit = 0.75
// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message,
// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689).
// Set by the controller; nil in a test or a tool.
var DeclarationMeasured func(ctx context.Context, node string, size, limit int64)
// payloadLimiter is a bus that knows its largest message.
type payloadLimiter interface{ MaxPayload() int64 }
// payloadLimitOf is the bus's largest message in bytes, or 0 when the bus does not say.
func payloadLimitOf(bus Bus) int64 {
if l, ok := bus.(payloadLimiter); ok {
return l.MaxPayload()
}
return 0
}
internal/link/declare_room_integration_test.go
-52
View File
@@ -1,52 +0,0 @@
//go:build integration
package link
import (
"context"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// Against a real bus whose largest message is small: the limit is read from the connection, the one that fits goes
// through, the one over is refused by us with its size and the limit, not by the client's bare error (issue 1689).
func TestADeclarationIsMeasuredAgainstTheRealBusLimit(t *testing.T) {
s := testbus.StartWithMaxPayload(t, 2048)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := js.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}}); err != nil {
t.Fatal(err)
}
bus := OverNATS{Conn: conn, JS: js}
if bus.MaxPayload() != 2048 {
t.Fatalf("the limit read from the connection is %d, expected 2048", bus.MaxPayload())
}
var told []int64
was := DeclarationMeasured
DeclarationMeasured = func(_ context.Context, _ string, size, limit int64) { told = append(told, size, limit) }
t.Cleanup(func() { DeclarationMeasured = was })
fits := []byte(`{"a":"` + strings.Repeat("x", 600) + `"}`)
if err := Declare(t.Context(), bus, fixedSigner{}, "novox", fits, 5*time.Second); err != nil {
t.Fatalf("a declaration under the limit: %v", err)
}
over := []byte(`{"a":"` + strings.Repeat("x", 1600) + `"}`)
err = Declare(t.Context(), bus, fixedSigner{}, "novox", over, 5*time.Second)
if err == nil || !strings.Contains(err.Error(), "2048 bytes") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("a declaration over the limit: %v", err)
}
if len(told) != 4 || told[1] != 2048 || told[2] <= 2048 {
t.Fatalf("measured %v", told)
}
}
internal/link/declare_room_test.go
-57
View File
@@ -1,57 +0,0 @@
package link
import (
"context"
"strings"
"testing"
"time"
)
type roomBus struct {
Bus
limit int64
sent [][]byte
}
func (b *roomBus) MaxPayload() int64 { return b.limit }
func (b *roomBus) PublishDeclaration(_ context.Context, _ string, body []byte) error {
b.sent = append(b.sent, body)
return nil
}
type fixedSigner struct{}
func (fixedSigner) Sign(context.Context, []byte) ([]byte, error) { return []byte("sig"), nil }
// A declaration is measured as it goes on the wire against the bus's own limit; one over it is refused saying size and
// limit, and what the measure sees is told to whoever raises a condition (issue 1689).
func TestADeclarationOverTheBusLimitIsRefusedSayingSizeAndLimit(t *testing.T) {
var seen []int64
was := DeclarationMeasured
DeclarationMeasured = func(_ context.Context, node string, size, limit int64) {
if node != "novox" || limit != 400 {
t.Errorf("measured %s against %d", node, limit)
}
seen = append(seen, size)
}
t.Cleanup(func() { DeclarationMeasured = was })
bus := &roomBus{limit: 400}
small := []byte(`{"a":"` + strings.Repeat("x", 100) + `"}`)
if err := Declare(t.Context(), bus, fixedSigner{}, "novox", small, time.Second); err != nil || len(bus.sent) != 1 {
t.Fatalf("a small declaration: %v, %d sent", err, len(bus.sent))
}
if int64(len(bus.sent[0])) != seen[0] {
t.Fatalf("measured %d bytes, sent %d", seen[0], len(bus.sent[0]))
}
big := []byte(`{"a":"` + strings.Repeat("x", 300) + `"}`)
err := Declare(t.Context(), bus, fixedSigner{}, "novox", big, time.Second)
if err == nil || !strings.Contains(err.Error(), "400 bytes") || !strings.Contains(err.Error(), "novox") ||
!strings.Contains(err.Error(), "over") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("an oversize declaration: %v", err)
}
if len(bus.sent) != 1 || len(seen) != 2 || seen[1] <= 400 {
t.Fatalf("it was sent (%d) or not measured (%v)", len(bus.sent), seen)
}
}
internal/link/enrolment.go
-49
View File
@@ -11,7 +11,6 @@ import (
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
@@ -268,16 +267,6 @@ var _ Enroller = Enrolment{}
// A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is
// its own account of its own machine, kept as a copy for recovery — so this writes it down and
// decides nothing from it.
// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not
// acted on (design 25 §3, window.go).
//
// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great
// many things that record reports have no idea what was sent — every test in this package among
// them. So the serving loop asks for this when the listener happens to be able to answer, and
// where it cannot, a report has nothing to be stale against and is simply acted on.
func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) {
return e.Inventory.Outstanding(ctx, node)
}
func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err error) {
// A store that could not be asked right now is said as such, so the report is kept for
@@ -444,11 +433,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
// arrives; the digest the mesh last sent decides nothing for it. **By digest, from an older
// node-engine**: its report claims no order, and the rule of issue 267 stands for it.
//
// And whether the node-engine reads an epoch in a declaration, from every account it gives: the
// mesh sends one only to a machine that said so, and one rolled back says so no longer.
if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil {
return false, err
}
if report.Ordered() {
account := AccountOf(report)
news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing,
@@ -467,17 +451,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
return false, err
}
} else {
if report.Declared != "" {
sent, err := e.Inventory.Outstanding(ctx, report.Node)
if err != nil {
return false, err
}
if Superseded(report.Declared, sent) {
log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
return false, e.Inventory.Seen(ctx, node.ID)
}
}
// **Whether this is news** is the store's answer: it holds the previous report, and a machine
// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
@@ -493,25 +466,3 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
}
return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied)
}
// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335).
func busyWords(busy []inventory.BusyWait) string {
var out []string
for _, b := range busy {
if b.State == inventory.BusyWaiting {
out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container,
b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339)))
}
}
return strings.Join(out, "; ")
}
// replacedAfterWaiting says a report replaced a container that had waited because it was busy.
func replacedAfterWaiting(busy []inventory.BusyWait) bool {
for _, b := range busy {
if b.State != inventory.BusyWaiting {
return true
}
}
return false
}
internal/link/generation_test.go
-48
View File
@@ -1,48 +0,0 @@
package link
import (
"encoding/json"
"testing"
)
// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it:
// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal
// of a declaration composed from an older generation than the machine applied.
func TestTheGenerationOnTheWire(t *testing.T) {
raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` +
`"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`)
var r Report
if err := json.Unmarshal(raw, &r); err != nil {
t.Fatal(err)
}
if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) ||
r.Sequence != 12 {
t.Fatalf("a node-engine's generation refusal reads as %+v", r)
}
// Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender.
if r.StaleRefusalOf() {
t.Errorf("a generation refusal reads as a stale writer's")
}
var older Report
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil {
t.Fatal(err)
}
if older.ReadsGeneration || older.OlderGeneration != nil {
t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older)
}
}
// 1145/AC3: a report's word that the node-engine reads a module per resource and a build per module, as mesh-host's
// report writes it: `reads_modules`, on every report of a node-engine that reads them.
func TestWhetherTheNodeEngineReadsModulesIsOnItsReport(t *testing.T) {
var reads, older Report
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":8,"reads_modules":true}`), &reads); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil {
t.Fatal(err)
}
if !reads.ReadsModules || older.ReadsModules {
t.Fatalf("1145/AC3: reads %v, an older node-engine reads %v", reads.ReadsModules, older.ReadsModules)
}
}
internal/link/heard_order_test.go
-22
View File
@@ -1,7 +1,6 @@
package link_test
import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/link"
@@ -25,27 +24,6 @@ func ordered(declared string, epoch, sequence, report int64, applied ...string)
// A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the
// order kept is cleared, and it is no longer sent an epoch.
// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose
// own report was lost is still counted from the machine's own tally.
func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) {
count := link.NewRefusalCount()
now := time.Now()
refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11},
OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3}
if !refusal.StaleRefusalOf() {
t.Fatal("a refusal naming the order it holds is not stale")
}
count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it
count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now})
count.Lifetime("anchor", refusal.RefusedOlder, now)
count.Lifetime("anchor", 5, now) // two more refused, their reports lost
got := count.Within(now.Add(-time.Minute))
if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" ||
got[1].Count != 1 || got[1].Receivers[0] != "anchor" {
t.Fatalf("the refusals by writer are %+v", got)
}
}
// countFor is how many refusals of a writer this process has heard in the last minute.
func countFor(writer string) int {
for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) {
internal/link/order.go
+9 -66
View File
@@ -2,68 +2,22 @@ package link
import "strconv"
// The order a declaration carries, and the order a report about one carries back (novox/hq to-be 45
// §6, ADR 0227 rule 2).
// The order of a node's facts (novox/hq ADR 0539 §9, to-be 45 §6, ADR 0227 rule 2).
//
// **This file is the controller's side of the contract with the node-engine for Phase 2**, and the one
// place it is written here. It mirrors mesh-host internal/link/messages.go (`Order`, and the report's
// `report_sequence`, `older_than`, `refused_older`) field for field and rule for rule, as Report has
// always mirrored the host's report: a key added on one side and not the other is a key the other side
// does not know exists.
//
// **The wire.**
//
// - A declaration's order is two top-level keys of its signed envelope, beside "declaration" and
// "resources": `sequence` (since issue 107) and `epoch` (new). Inside the signed bytes, so a message
// cannot be given a newer order than the controller gave it.
// - A report carries the order of the declaration it is about as the same two top-level keys, beside
// "declared"; `report_sequence`, the node-engine's own number for the report; `older_than`, on a
// refusal of a declaration older than one it applied, the order of the one it holds; and
// `refused_older`, how many it has refused so, ever, on every report.
//
// Every key is optional and absent when zero, and zero is "no order claimed", never "first". So each
// side reads the other's older shape:
//
// - **An older node-engine with this controller.** It decodes a declaration strictly and refuses a key
// it does not know, whole — so `epoch` is sent only to a machine whose latest report carried a
// `report_sequence`, which a node-engine that reads the epoch always does. Until then it is sent the
// sequence alone, as today; its reports carry no report sequence and are judged by the digest they
// name, as today (issue 267).
// - **A newer node-engine with an older controller.** It is sent no epoch, which claims none: it
// refuses nothing by epoch. The keys it adds to a report are fields the older controller ignores.
// - **A controller rolled back to a build without the lease** sends no epoch again and is not refused
// for it: a node-engine refuses by epoch only when both declarations claim one. That gives up the
// epoch's protection while such a build runs — the price of a rollback that cannot strand every
// machine.
//
// **Epoch** is the controller lease's epoch (to-be 45 §6): the lease bucket's revision at which the
// instance that composed the declaration took the lease. It only grows: a later holder's is higher, and
// the controller never issues one at or under the highest it has issued (internal/lease, the floor).
// **Sequence** is the declaration's number for that machine, one higher every send, taken from the
// controller's store before the declaration is composed (issues 107, 204).
// A node-engine says its facts (profile, firewall found, reachable, tunnel, witness, rollbacks, rekey, health) in a report
// on the bus, and orders them itself: `epoch` and `sequence` are the node-engine's own (Writer `node-engine`), and
// `report_sequence` numbers each report it makes. The controller keeps the newest account of a machine and refuses an older
// one, said and counted (rule 2): by epoch, then sequence, then report sequence, each compared only where both claim one.
// Zero is "no order claimed", never "first". The controller sends a machine no declaration to be ordered by: the order here
// is only ever the machine's own.
// Order is where a declaration stands among everything the mesh has sent a machine: the lease epoch it
// was sent under and its sequence. Zero in either claims none.
// Order is where a node-engine's account stands among its own: the epoch and sequence the node-engine counts. Zero in either
// claims none.
type Order struct {
Epoch int64 `json:"epoch,omitempty"`
Sequence int64 `json:"sequence,omitempty"`
}
// Older is the node-engine's rule, as mesh-host states it: a declaration of this order is older than
// one of order `than` the machine applied **only when both claim an epoch** — and then when its epoch
// is lower, or its epoch is the same and its sequence lower (both claimed). A higher epoch is a new lease
// holder and is never older, whatever its sequence. Here so the controller's tests state what the
// machines will do with what it sends.
func (o Order) Older(than Order) bool {
if o.Epoch <= 0 || than.Epoch <= 0 {
return false
}
if o.Epoch != than.Epoch {
return o.Epoch < than.Epoch
}
return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence
}
// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a Stringer
// promoted onto every report would print each one as its order alone.
func (o Order) Words() string {
@@ -91,17 +45,6 @@ func AccountOf(r Report) Account { return Account{Order: r.Order, ReportSequence
// before (issue 267).
func (r Report) Ordered() bool { return r.ReportSequence > 0 }
// ReadsEpoch says the node-engine that made the report takes an epoch in a declaration: one that orders
// its reports does (mesh-host: "its presence also says this host reads a declaration's epoch").
func (r Report) ReadsEpoch() bool { return r.ReportSequence > 0 }
// StaleRefusalOf says a report refuses a declaration older than one the machine applied: the node-engine
// names the order it holds (`older_than`), or, from a node-engine older than that, says so in the words
// of its sequence refusal.
func (r Report) StaleRefusalOf() bool {
return r.OlderThan != nil || (r.Refused != "" && IsStaleRefusal(r.Refused))
}
// OlderThan is the controller's rule (to-be 45 §6): whether this account is older than the one kept for
// the machine — **by epoch, then sequence, then report sequence**, each compared only where both claim
// one. An account of a declaration of an older epoch is refused; of an older sequence, refused; an older
internal/link/order_test.go
+4 -36
View File
@@ -5,34 +5,8 @@ import (
"testing"
)
// The contract of order.go, case by case: what a node-engine refuses of a declaration (the rule
// mesh-host states, restated so the controller's tests say what the machines do with what it sends),
// what the controller refuses of a report, and the bytes on the wire both ways.
func TestADeclarationIsOlderOnlyWhenBothClaimAnEpoch(t *testing.T) {
for _, c := range []struct {
name string
arriving Order
held Order
older bool
}{
{"a newer sequence of the same epoch", Order{57, 12}, Order{57, 11}, false},
{"the same declaration again (a reconcile)", Order{57, 12}, Order{57, 12}, false},
{"a lower sequence of the same epoch", Order{57, 11}, Order{57, 12}, true},
// Issue 204: a controller that lost its lease goes on sending.
{"an older epoch, whatever its sequence", Order{41, 99}, Order{57, 12}, true},
{"a newer epoch, whatever its sequence", Order{57, 3}, Order{41, 99}, false},
{"no epoch arriving: a rolled-back controller is not stranded", Order{0, 11}, Order{57, 12}, false},
{"no epoch held: what the machine held before any lease", Order{57, 11}, Order{0, 12}, false},
{"no order at all", Order{}, Order{57, 12}, false},
} {
t.Run(c.name, func(t *testing.T) {
if got := c.arriving.Older(c.held); got != c.older {
t.Fatalf("%s against %s: older %v, want %v", c.arriving.Words(), c.held.Words(), got, c.older)
}
})
}
}
// The contract of order.go, case by case: what the controller refuses of a node-engine's facts (the node-engine's own order)
// and the bytes on the wire.
func TestAnAccountIsOlderByEpochThenSequenceThenReportSequence(t *testing.T) {
for _, c := range []struct {
@@ -78,7 +52,7 @@ func TestTheOrderOnTheWire(t *testing.T) {
r.RefusedOlder != 3 {
t.Fatalf("a node-engine's stale refusal reads as %+v", r)
}
if !r.Ordered() || !r.ReadsEpoch() || !r.StaleRefusalOf() {
if !r.Ordered() {
t.Fatalf("a node-engine that orders its reports reads as one that does not: %+v", r)
}
@@ -87,15 +61,9 @@ func TestTheOrderOnTheWire(t *testing.T) {
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"declared":"d1"}`), &older); err != nil {
t.Fatal(err)
}
if older.Ordered() || older.ReadsEpoch() || older.StaleRefusalOf() {
if older.Ordered() {
t.Fatalf("an older node-engine's report reads as ordered: %+v", older)
}
// Its stale refusal, in the words it has always used, is still one.
older.Refused = "this declaration " + StaleRefusal + ": it is sequence 3"
if !older.StaleRefusalOf() {
t.Fatal("an older node-engine's refusal of an older sequence does not read as stale")
}
// And a report with no order puts no order key on the wire.
raw, err := json.Marshal(Report{Node: "anchor", Declared: "d1"})
if err != nil {
internal/link/receive.go
-12
View File
@@ -121,15 +121,3 @@ type Inbound interface {
// Close lets go of whatever the implementation holds.
Close()
}
// Outstanding answers which declaration the mesh last sent a node — the digest, not the
// declaration.
//
// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh
// has already moved past is not worth holding a slot in the window that a current message needs.
// It is a separate interface from Listener rather than a method on it, because a controller that
// only publishes needs neither and something that records reports need not also be able to say
// what was sent.
type Outstanding interface {
Outstanding(ctx context.Context, node string) (string, error)
}
internal/link/receive_nats_integration_test.go
-19
View File
@@ -102,25 +102,6 @@ func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T
})
}
func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) {
js := aBus(t)
store := &sentAndHeardSafely{sent: "d2"}
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "the superseded report leaving the stream", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
if store.count() != 0 {
t.Fatalf("a report about a superseded declaration was acted on")
}
}
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
js := aBus(t)
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
internal/link/serve.go
+1 -59
View File
@@ -292,26 +292,6 @@ func (s *Server) giveUpAfter() time.Duration {
return s.giveUp
}
// outstanding is the digest of the declaration the mesh last sent a node.
//
// Nothing is guessed when it cannot be answered: a listener that keeps no record of what was sent
// gives a report nothing to be stale against, and a store that cannot be read will hold the report
// anyway, so there is nothing for the check to decide.
func (s *Server) outstanding(ctx context.Context, node string) string {
if node == "" {
return ""
}
asks, ok := s.listener.(Outstanding)
if !ok {
return ""
}
digest, err := asks.Outstanding(ctx, node)
if err != nil {
return ""
}
return digest
}
// toolsHeartbeat records that a machine's node tools were heard from (novox/hq to-be 45 S11), and
// nothing else: in this process's memory, for the watchdog — the next one is a minute away.
func (s *Server) toolsHeartbeat(m Control) {
@@ -343,25 +323,8 @@ func (s *Server) reported(ctx context.Context, m Control) {
what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared)
if s.listener != nil {
// **Asked before the store, not after** (design 25 §3). A report about a declaration the
// mesh has moved past would otherwise wait out a restarting store to be written and then
// overwrite what the node is doing now — and on the bus being built, where the holding is
// the server's, it comes back after the newer was applied whatever the controller does.
outstanding := s.outstanding(ctx, report.Node)
declaredIn := staleAgainst(report)
if report.Ordered() {
// Kept by its order, not by the digest the mesh last sent (novox/hq to-be 45 §6): the
// listener refuses an older account under the one rule, whatever it is about.
declaredIn = ""
}
if Superseded(declaredIn, outstanding) {
s.log.Printf("set aside %s: the mesh has moved past that declaration", what)
_ = m.Took()
return
}
news, err := s.listener.Heard(context.Background(), report)
switch s.decide(ctx, m, what, declaredIn, outstanding, err) {
switch s.decide(ctx, m, what, "", "", err) {
case Hold:
// Held, not settled, while the store cannot take it: the node reports an apply once,
// and a report lost here is a node the mesh never hears from again — the store
@@ -396,27 +359,6 @@ func (s *Server) reported(ctx context.Context, m Control) {
_ = m.Took()
}
// staleAgainst is the declaration a report may be judged stale against, and it is empty for a
// report that is not only an account of an apply.
//
// **A report carries two different things, and only one of them is about a declaration.** What the
// node applied is; what the machine *is* — the tunnel it took over, the ports its own bundle holds,
// what an adopted node found and is keeping, a node moving its overlay key — is not. Those reach
// the mesh on a report because a report is the message a node sends, and nowhere else: a rekey
// dropped as stale is a node whose overlay key never moves, and no retry is coming, because the
// node said it once.
//
// So staleness is asked only of a report that is purely an apply's account. The rest is acted on
// whenever it arrives, which is the behaviour the mesh has had all along.
func staleAgainst(report Report) string {
if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 ||
report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 ||
len(report.Outward) > 0 {
return ""
}
return report.Declared
}
func (s *Server) enrolling(ctx context.Context, m Control) {
reply := EnrolReply{Refusal: "that token cannot be used"}
internal/link/signer.go
+12
View File
@@ -0,0 +1,12 @@
package link
import "context"
// Signer is whatever holds the control plane's signing key.
type Signer interface {
Sign(ctx context.Context, message []byte) ([]byte, error)
}
// ActingGate is what every act passes before it is made (novox/hq to-be 45 §6): whether this process may act under the
// controller's lease. Set by the controller; nil passes every act — a test, a tool.
var ActingGate func(ctx context.Context) error
internal/link/stale_report_test.go
-123
View File
@@ -1,123 +0,0 @@
package link
import (
"context"
"errors"
"testing"
)
// Supersession as a check rather than a memory (design 25 §3).
//
// Holding a message in memory let the controller drop an older report when a newer one for the
// same node arrived. On the bus being built the message belongs to the server and comes back
// whatever happened meanwhile — so the older report is redelivered *after* the newer was applied,
// and acting on it would undo the newer.
//
// The answer was already in the message: a report carries the digest of the declaration it is
// about, so "is this the past?" is a question the message answers.
// sentAndHeard records reports and knows what was last sent, which is the pair the check needs.
type sentAndHeard struct {
sent string
heard []Report
err error
}
func (s *sentAndHeard) Heard(_ context.Context, r Report) (bool, error) {
if s.err != nil {
return false, s.err
}
s.heard = append(s.heard, r)
return true, nil
}
func (s *sentAndHeard) Outstanding(context.Context, string) (string, error) { return s.sent, nil }
// A report about a declaration the mesh has moved past is settled and not acted on. Settled rather
// than dropped, because there is nothing wrong with the message — it is simply the past, and
// redelivering it for ever is worse than letting it go.
func TestAReportAboutASupersededDeclarationIsNotActedOn(t *testing.T) {
store := &sentAndHeard{sent: "d2"}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1")))
if len(store.heard) != 0 {
t.Fatalf("a report about a superseded declaration was acted on: %+v", store.heard)
}
if !to.acked {
t.Fatalf("a superseded report was not settled, so it comes back for ever: %+v", *to)
}
}
// The report about the declaration that *is* outstanding is acted on, and so is one from a node
// the mesh has no digest for — an older host that says nothing about which declaration it applied
// has nothing to be judged against, and refusing it would silence every node built before reports
// carried the digest.
func TestAReportAboutTheOutstandingDeclarationIsActedOn(t *testing.T) {
for _, c := range []struct{ what, sent, declared string }{
{"the one outstanding", "d2", "d2"},
{"a report that says nothing about which", "d2", ""},
{"a node nothing was ever sent", "", "d1"},
} {
store := &sentAndHeard{sent: c.sent}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", c.declared)))
if len(store.heard) != 1 || !to.acked {
t.Errorf("%s: was not acted on and acknowledged: heard %+v, settled %+v",
c.what, store.heard, *to)
}
}
}
// **Staleness is decided before the store is waited on**, not after: a redelivery that lost its
// race is not worth holding a slot in the window that a current message needs.
func TestASupersededReportIsNotHeldForTheStore(t *testing.T) {
store := &sentAndHeard{sent: "d2", err: errors.Join(ErrTryAgain, errors.New("starting up"))}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1")))
if !to.acked || len(in.held) != 0 {
t.Fatalf("a superseded report waited for the store: %+v, %d held", *to, len(in.held))
}
}
// **Half of a report is not about a declaration, and that half is never stale.**
//
// What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted
// node found and is keeping, a node moving its overlay key — reaches the mesh on a report and
// nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is
// coming, because the node said it once. So a report carrying any of these is acted on whenever it
// arrives, however far the mesh has moved on.
func TestAReportCarryingWhatOnlyTheNodeKnowsIsActedOnHoweverOldItIs(t *testing.T) {
for _, c := range []struct {
what string
report Report
}{
{"a rekey", Report{Node: "anchor", Declared: "d1",
Rekey: &Rekey{Previous: "k1", OverlayKey: "k2"}}},
{"the tunnel it carried", Report{Node: "anchor", Declared: "d1",
Tunnel: &CarriedTunnel{Interface: "wg0", State: "taken"}}},
{"what an adopted node holds", Report{Node: "anchor", Declared: "d1",
Held: []Held{{ID: "conf", Module: "web", Kind: "file"}}}},
{"the firewall it found", Report{Node: "anchor", Declared: "d1", Firewall: "ufw"}},
{"what is reachable on it", Report{Node: "anchor", Declared: "d1",
Reachable: []Reach{{Protocol: "tcp", Port: 443}}}},
{"the ports its own bundle holds", Report{Node: "anchor", Declared: "d1",
Carried: []int{5432}}},
} {
store := &sentAndHeard{sent: "d9"}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, c.report))
if len(store.heard) != 1 {
t.Errorf("%s was set aside as stale, and the mesh will never hear it again: %+v",
c.what, *to)
}
}
}