Apply through one queue, and order what is applied and reported (hq to-be 45 Phase 2)
A delivery and the five-minute reconcile were two paths that applied, ordered only by a lock, and each order it allowed was met live (issues 257, 261, 267). Now both only enqueue: one worker takes the newest declaration held when it starts, applies it once and makes one report, and reports leave in the order they are made. A declaration may carry the controller's lease epoch beside its sequence; one older than what this node applied is refused before anything is touched, counted, logged and reported. A report carries the declaration's epoch and sequence and the host's own report sequence, kept on disk so it goes on increasing across restarts and self-updates. Without an epoch, today's behaviour stands.
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be
|
||||
// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node
|
||||
// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before
|
||||
// the machine is read or touched, with a report naming what was refused and what is held, and the
|
||||
// kept declaration left as it was.
|
||||
func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
_, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(dir, "a.conf")
|
||||
declared := func(epoch, sequence int64) store.Declared {
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence,
|
||||
"resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
|
||||
}
|
||||
held := declared(57, 3)
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stale := declared(41, 12)
|
||||
report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil)
|
||||
if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) ||
|
||||
report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) ||
|
||||
report.Refused == "" {
|
||||
t.Fatalf("the refusal does not name what was refused and what is held: %+v", report)
|
||||
}
|
||||
if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("the refused declaration touched the machine")
|
||||
}
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(opts.state))
|
||||
if err != nil || string(kept.Declaration) != string(held.Declaration) {
|
||||
t.Fatal("the refused declaration replaced what this node kept")
|
||||
}
|
||||
}
|
||||
|
||||
// The queue's counts are kept beside the state and read back by the next host; unreadable is an
|
||||
// error, never zero.
|
||||
func TestTheNumbersSurviveTheHost(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 {
|
||||
t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err)
|
||||
}
|
||||
if err := numbersBeside(state).Save(41, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 {
|
||||
t.Fatalf("read back %d, %d, %v", sequence, refused, err)
|
||||
}
|
||||
if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := numbersBeside(state).Read(); err == nil {
|
||||
t.Fatal("unreadable numbers were read as zero")
|
||||
}
|
||||
}
|
||||
+139
-74
@@ -458,10 +458,10 @@ func short(digest string) string {
|
||||
// them again — and everything the mesh declared read as no longer declared and removed. Even the
|
||||
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
|
||||
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
|
||||
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, sequence int64) error {
|
||||
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, order link.Order) error {
|
||||
switch from {
|
||||
case fromDeclared:
|
||||
return refuseOlder(kept, keptErr, sequence)
|
||||
return refuseOlder(kept, keptErr, order)
|
||||
case fromBundle:
|
||||
if known.Genesis == nil || known.Genesis.Digest == digest {
|
||||
return nil
|
||||
@@ -558,7 +558,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
|
||||
if err := apply.CheckMode(known, d); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := refuseStale(known, kept, keptErr, digest, from, d.Sequence); err != nil {
|
||||
if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1057,6 +1057,14 @@ func runLink(ctx context.Context, opts options) error {
|
||||
aside, standAside := context.WithCancel(ctx)
|
||||
defer standAside()
|
||||
stoodAside := false
|
||||
membership := link.Membership{
|
||||
Node: mine.Node,
|
||||
Broker: mine.Membership.Broker,
|
||||
Fingerprint: mine.Membership.Fingerprint,
|
||||
Password: mine.Membership.Password,
|
||||
Transport: mine.Membership.Transport,
|
||||
Signer: mine.Membership.Signer,
|
||||
}
|
||||
|
||||
applier := func(ctx context.Context, raw, signature []byte) link.Report {
|
||||
report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
|
||||
@@ -1084,40 +1092,44 @@ func runLink(ctx context.Context, opts options) error {
|
||||
}
|
||||
|
||||
// Two things at once, and the second is what makes disconnection ordinary. The link brings
|
||||
// new declarations; this holds the machine in the last one whether the link is up or not. A
|
||||
// laptop shut for a week comes back and reconciles — it does not come back and ask what it is
|
||||
// (novox/hq ADR 0004).
|
||||
// Reports a reconcile has to make unasked — what an adopted node holds changed, or its
|
||||
// firewall did — go out over the link when it is up (novox/hq ADR 0100).
|
||||
outbox := make(chan link.Unasked, 1)
|
||||
// new declarations; the reconcile holds the machine in the last one whether the link is up or
|
||||
// not. A laptop shut for a week comes back and reconciles — it does not come back and ask what
|
||||
// it is (novox/hq ADR 0004).
|
||||
//
|
||||
// **Both only enqueue** (novox/hq to-be 45 §6). One worker applies, the newest declaration held
|
||||
// when it starts, once, and makes one report; a reconcile due while a delivery waits is that
|
||||
// delivery's apply. Reports a reconcile has to make unasked — what an adopted node holds changed,
|
||||
// its firewall, its outward links — are said by the same worker, in the order they are made
|
||||
// (novox/hq ADR 0100, issue 267).
|
||||
watch := &adoptionWatch{}
|
||||
applier = watch.noting(applier)
|
||||
go holdTheMachine(ctx, opts, mine, say, sched, func(r link.Report) {
|
||||
if !watch.differs(r) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-outbox:
|
||||
// An older one nobody has published yet; this one says everything it did.
|
||||
default:
|
||||
}
|
||||
queue := &link.Queue{
|
||||
Membership: membership,
|
||||
Apply: applier,
|
||||
Reconcile: reconcileKept(opts, mine.Membership.Signer, sched, say),
|
||||
News: func(r link.Report) bool { return worthSaying(r) && watch.differs(r) },
|
||||
// Counted as said only once the broker has taken it: queued and lost — the link down, the
|
||||
// publish refused — the change would never be said again (novox/hq ADR 0100).
|
||||
outbox <- link.Unasked{Report: r, Done: func(published bool) {
|
||||
if published {
|
||||
watch.said(r)
|
||||
}
|
||||
}}
|
||||
})
|
||||
Heard: watch.said,
|
||||
Unsaid: unsaidBeside(opts.state),
|
||||
Numbers: numbersBeside(opts.state),
|
||||
Say: say,
|
||||
Timeout: opts.timeout,
|
||||
}
|
||||
worker := make(chan struct{})
|
||||
go func() {
|
||||
defer close(worker)
|
||||
queue.Run(aside)
|
||||
}()
|
||||
// **A host starting is a reason to reconcile** (to-be 45 §6: the self-update hand-over is one of
|
||||
// the four): its scheduled steps are armed from the declaration the node kept by the first apply,
|
||||
// and a successor that waited five minutes for it left them unarmed for five.
|
||||
queue.ReconcileDue()
|
||||
go holdTheMachine(ctx, queue)
|
||||
|
||||
held := link.HoldRoused(aside, link.Membership{
|
||||
Node: mine.Node,
|
||||
Broker: mine.Membership.Broker,
|
||||
Fingerprint: mine.Membership.Fingerprint,
|
||||
Password: mine.Membership.Password,
|
||||
Transport: mine.Membership.Transport,
|
||||
Signer: mine.Membership.Signer,
|
||||
}, applier, say, opts.timeout, rousedBySignal(ctx), outbox, unsaidBeside(opts.state))
|
||||
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
|
||||
// The act in hand finishes and is kept before this process exits: an apply that stood aside with
|
||||
// no link open has its report only in the unsaid store, and only once the worker has written it.
|
||||
<-worker
|
||||
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
|
||||
// before it has run once. The context this returns on was cancelled deliberately, so its error
|
||||
// is not a fault to report.
|
||||
@@ -1169,6 +1181,23 @@ func (u unsaidFile) Pending() (link.Report, bool, error) {
|
||||
return r, r.Declared != "", nil
|
||||
}
|
||||
|
||||
// numbersFile keeps the queue's report sequence and its count of declarations refused as older beside
|
||||
// the node's state (novox/hq to-be 45 §6), where a successor reads them and goes on from there.
|
||||
type numbersFile struct{ path string }
|
||||
|
||||
func numbersBeside(statePath string) numbersFile {
|
||||
return numbersFile{path: store.NumbersPath(statePath)}
|
||||
}
|
||||
|
||||
func (n numbersFile) Read() (int64, int64, error) {
|
||||
kept, err := store.ReadNumbers(n.path)
|
||||
return kept.ReportSequence, kept.RefusedOlder, err
|
||||
}
|
||||
|
||||
func (n numbersFile) Save(reportSequence, refusedOlder int64) error {
|
||||
return store.SaveNumbers(n.path, store.Numbers{ReportSequence: reportSequence, RefusedOlder: refusedOlder})
|
||||
}
|
||||
|
||||
// adoptionWatch remembers what the node last said about what it holds and its firewall, so a
|
||||
// reconcile speaks unasked only when that changed.
|
||||
type adoptionWatch struct {
|
||||
@@ -1232,15 +1261,6 @@ func (w *adoptionWatch) changed(r link.Report) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// noting wraps the applier, so a report the link publishes after a delivery counts as said.
|
||||
func (w *adoptionWatch) noting(apply link.Applier) link.Applier {
|
||||
return func(ctx context.Context, raw, signature []byte) link.Report {
|
||||
r := apply(ctx, raw, signature)
|
||||
w.changed(r)
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// rousedBySignal is the machine telling this process that its link is probably stale.
|
||||
//
|
||||
// **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would
|
||||
@@ -1285,43 +1305,44 @@ func rousedBySignal(ctx context.Context) link.Roused {
|
||||
// changed it, and then for ever.
|
||||
const ReconcileEvery = 5 * time.Minute
|
||||
|
||||
func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, say link.Announce,
|
||||
sched *apply.Scheduler, publish func(link.Report)) {
|
||||
// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq
|
||||
// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is
|
||||
// waiting is that delivery's apply.
|
||||
func holdTheMachine(ctx context.Context, queue *link.Queue) {
|
||||
ticker := time.NewTicker(ReconcileEvery)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
queue.ReconcileDue()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report, err := reapplyKept(ctx, opts, mine.Membership.Signer, sched, say)
|
||||
// reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it
|
||||
// was last told, read once it is this act's turn, and say what did not go.
|
||||
//
|
||||
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or its firewall
|
||||
// changed, because that is how a predecessor still writing is caught (novox/hq ADR 0100); and on any
|
||||
// node when it can say which links face outside (ADR 0140) — a converged node holds nothing and found
|
||||
// no firewall, so without that it could speak only in reply to a declaration, while the mesh composes
|
||||
// no declaration for a node that has not said which links face outside. Whether a report is news is
|
||||
// the queue's News (worthSaying and the watch), so an unchanged answer costs one comparison every
|
||||
// reconcile and nothing on the bus.
|
||||
func reconcileKept(opts options, signer ed25519.PublicKey, sched *apply.Scheduler,
|
||||
say link.Announce) link.Reconcile {
|
||||
return func(ctx context.Context) (link.Report, bool) {
|
||||
report, err := reapplyKept(ctx, opts, signer, sched, say)
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
// Nothing to hold this machine to yet. Ordinary on a node that has enrolled and not
|
||||
// been assigned anything.
|
||||
continue
|
||||
return link.Report{}, false
|
||||
}
|
||||
if err != nil {
|
||||
say("cannot re-apply what this node was told: " + err.Error())
|
||||
continue
|
||||
}
|
||||
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
|
||||
// its firewall changed, because that is how a predecessor still writing is caught
|
||||
// (novox/hq ADR 0100); publish decides whether anything did.
|
||||
//
|
||||
// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
|
||||
// converged node holds nothing and found no firewall, so this gate closed on it and the
|
||||
// node could speak only in reply to a declaration — while the mesh composes no declaration
|
||||
// for a node that has not said which links face outside. A machine waiting for a push that
|
||||
// was waiting for the machine, and measured: three converged machines sat silent while the
|
||||
// control plane refused to send them a filter.
|
||||
//
|
||||
// Offered, not published: whether it is news is still the watch's to decide, so an
|
||||
// unchanged answer costs one comparison every reconcile and nothing on the bus.
|
||||
if publish != nil && worthSaying(report) {
|
||||
publish(report)
|
||||
return link.Report{}, false
|
||||
}
|
||||
switch {
|
||||
case report.Refused != "":
|
||||
@@ -1329,6 +1350,7 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
|
||||
case len(report.Failed) > 0:
|
||||
say(fmt.Sprintf("holding this machine: %d applied, and %v", len(report.Applied), report.Failed))
|
||||
}
|
||||
return report, true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1367,13 +1389,13 @@ func announceOr(say link.Announce) func(string) {
|
||||
|
||||
// applying serialises applies within this process.
|
||||
//
|
||||
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
|
||||
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
|
||||
// one that saves last writes a state read before the other acted — losing what the first recorded:
|
||||
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
|
||||
// and its record another, which is the fault every read-back in this package exists to prevent.
|
||||
// Across processes — `reconcile` run by hand beside this service — the lock beside the state does
|
||||
// the same (store.Lock).
|
||||
// **One thing applies here: the apply queue's worker** (novox/hq to-be 45 §6), for a delivery and for
|
||||
// the reconcile alike. This lock was the only order between two that applied — the link and the
|
||||
// reconcile loop — and each order it allowed was met as a fault (issues 257, 261, 267); the queue is
|
||||
// the order now. It stays as the guard for what may be added beside the worker: two applies
|
||||
// interleaved lose what one of them recorded — a hold, the firewall found here, a resource just
|
||||
// applied — and the machine is then one thing and its record another. Across processes — `reconcile`
|
||||
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
||||
var applying sync.Mutex
|
||||
|
||||
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
|
||||
@@ -1418,6 +1440,21 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
order := link.Order{Epoch: declared.Epoch, Sequence: declared.Sequence}
|
||||
|
||||
// **Older than what this node applied is refused, before anything is read from the machine**
|
||||
// (novox/hq to-be 45 §6, ADR 0227 rule 2): a controller that lost its lease and goes on sending.
|
||||
// Read under the lock, so what it is compared with is what the last apply kept. Only a delivery:
|
||||
// what a reconcile applies is what was kept.
|
||||
if signed != nil {
|
||||
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
|
||||
if held, older := olderThanKept(kept, keptErr, order); older {
|
||||
return link.Report{Declared: digestOf(raw), Order: order, OlderThan: &held,
|
||||
Refused: fmt.Sprintf("this declaration was sent under lease epoch %d, sequence %d, and this "+
|
||||
"node has applied epoch %d, sequence %d — from the controller holding the lease now. "+
|
||||
"Refused whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)}
|
||||
}
|
||||
}
|
||||
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
@@ -1490,7 +1527,7 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
sched.Sync(declared, held)
|
||||
}
|
||||
|
||||
report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: runningVersion(),
|
||||
report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(),
|
||||
Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
|
||||
// Which of this machine's links face outside, for the filter the mesh writes around them
|
||||
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
|
||||
@@ -1720,12 +1757,23 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
|
||||
// sent, and one kept with no sequence is one this host received before it understood them; in either
|
||||
// case there is no order to compare, and refusing on a guess would strand the node the moment the
|
||||
// controller is older than the host. Equal is the same declaration again, which reconciling is for.
|
||||
func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
|
||||
//
|
||||
// **And by the lease epoch first** (novox/hq to-be 45 §6): a declaration sent under an older epoch than
|
||||
// the one kept here is a controller that lost its lease, whatever its sequence — the same rule the
|
||||
// link's deliveries are held to (olderThanKept). Where both claim an epoch it alone decides, because a
|
||||
// new lease holder's sequence says nothing against the last one's.
|
||||
func refuseOlder(kept store.Declared, keptErr error, order link.Order) error {
|
||||
if held, older := olderThanKept(kept, keptErr, order); older {
|
||||
return fmt.Errorf("this declaration was sent under lease epoch %d, sequence %d, and the one kept "+
|
||||
"here under epoch %d, sequence %d: a controller that no longer holds the lease sent it. Refused "+
|
||||
"whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)
|
||||
}
|
||||
sequence := order.Sequence
|
||||
if sequence == 0 || keptErr != nil {
|
||||
return nil
|
||||
}
|
||||
last, err := declaration.ParseTrusted(kept.Declaration)
|
||||
if err != nil || last.Sequence == 0 {
|
||||
if err != nil || last.Sequence == 0 || (order.Epoch > 0 && last.Epoch > 0) {
|
||||
return nil
|
||||
}
|
||||
if sequence < last.Sequence {
|
||||
@@ -1737,6 +1785,23 @@ func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// olderThanKept says whether a declaration of this order is older than the one this node kept — the
|
||||
// last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept
|
||||
// declaration that cannot be read claims no order: what it is compared with is unknown, and refusing
|
||||
// on a guess would hold the machine off everything the mesh sends until a person looked. That kept
|
||||
// file is read, and refused by name, by the next reconcile.
|
||||
func olderThanKept(kept store.Declared, keptErr error, order link.Order) (link.Order, bool) {
|
||||
if keptErr != nil || order.Epoch <= 0 {
|
||||
return link.Order{}, false
|
||||
}
|
||||
last, err := declaration.ParseTrusted(kept.Declaration)
|
||||
if err != nil {
|
||||
return link.Order{}, false
|
||||
}
|
||||
held := link.Order{Epoch: last.Epoch, Sequence: last.Sequence}
|
||||
return held, order.Older(held)
|
||||
}
|
||||
|
||||
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
|
||||
// report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161).
|
||||
func profileAsReported(detected profile.Profile) map[string]any {
|
||||
|
||||
@@ -189,13 +189,15 @@ func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) {
|
||||
// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue
|
||||
// tells the watch every report the broker took (link.Queue.Heard).
|
||||
func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
|
||||
applier := w.noting(func(context.Context, []byte, []byte) link.Report { return report })
|
||||
applier(context.Background(), nil, nil)
|
||||
heard := w.said
|
||||
heard(report)
|
||||
if w.changed(report) {
|
||||
t.Error("a reconcile repeated what the link had just published")
|
||||
t.Error("a reconcile repeated what the mesh had just heard")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
@@ -13,9 +14,15 @@ import (
|
||||
// "older" could not.
|
||||
|
||||
func keptWith(t *testing.T, sequence int64) store.Declared {
|
||||
t.Helper()
|
||||
return keptAt(t, link.Order{Sequence: sequence})
|
||||
}
|
||||
|
||||
func keptAt(t *testing.T, order link.Order) store.Declared {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": sequence,
|
||||
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence,
|
||||
"epoch": order.Epoch,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -24,7 +31,7 @@ func keptWith(t *testing.T, sequence int64) store.Declared {
|
||||
}
|
||||
|
||||
func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
|
||||
err := refuseOlder(keptWith(t, 7), nil, 5)
|
||||
err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5})
|
||||
if err == nil {
|
||||
t.Fatal("sequence 5 was accepted over a kept 7")
|
||||
}
|
||||
@@ -34,11 +41,11 @@ func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
|
||||
if err := refuseOlder(keptWith(t, 7), nil, 8); err != nil {
|
||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil {
|
||||
t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
|
||||
}
|
||||
// Equal is the same declaration again, which reconciling is for.
|
||||
if err := refuseOlder(keptWith(t, 7), nil, 7); err != nil {
|
||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil {
|
||||
t.Fatalf("the same sequence was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -46,13 +53,43 @@ func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
|
||||
func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
|
||||
// An older controller sends none; a host that received before it understood them kept none.
|
||||
// Refusing on a guess would strand a node the moment the controller is older than the host.
|
||||
if err := refuseOlder(keptWith(t, 7), nil, 0); err != nil {
|
||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil {
|
||||
t.Fatalf("a declaration claiming no order was refused: %v", err)
|
||||
}
|
||||
if err := refuseOlder(keptWith(t, 0), nil, 3); err != nil {
|
||||
if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil {
|
||||
t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
|
||||
}
|
||||
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, 3); err != nil {
|
||||
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil {
|
||||
t.Fatalf("a first declaration was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost
|
||||
// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence.
|
||||
func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) {
|
||||
kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3})
|
||||
err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12})
|
||||
if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") {
|
||||
t.Fatalf("an older epoch was not refused by name: %v", err)
|
||||
}
|
||||
if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil {
|
||||
t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err)
|
||||
}
|
||||
if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil {
|
||||
t.Fatal("a lower sequence under the same epoch was accepted")
|
||||
}
|
||||
// A controller with no lease — older, or rolled back to a build from before it — is today's
|
||||
// behaviour: no epoch compared.
|
||||
if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil {
|
||||
t.Fatalf("a declaration with no epoch was refused for one: %v", err)
|
||||
}
|
||||
if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older ||
|
||||
held != (link.Order{Epoch: 57, Sequence: 3}) {
|
||||
t.Fatalf("olderThanKept said %v, holding %+v", older, held)
|
||||
}
|
||||
// What is kept cannot be read: no order to compare, so nothing is refused on a guess.
|
||||
if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil,
|
||||
link.Order{Epoch: 41, Sequence: 1}); older {
|
||||
t.Fatal("refused against a kept declaration nobody can read")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user