Apply through one queue, and order what is applied and reported (hq to-be 45 Phase 2)

A delivery and the five-minute reconcile were two paths that applied, ordered only by a lock, and
each order it allowed was met live (issues 257, 261, 267). Now both only enqueue: one worker takes
the newest declaration held when it starts, applies it once and makes one report, and reports leave
in the order they are made.

A declaration may carry the controller's lease epoch beside its sequence; one older than what this
node applied is refused before anything is touched, counted, logged and reported. A report carries
the declaration's epoch and sequence and the host's own report sequence, kept on disk so it goes on
increasing across restarts and self-updates. Without an epoch, today's behaviour stands.
This commit is contained in:
jochen
2026-10-06 11:54:10 +02:00
parent d7d93f57c5
commit 31804bd8c2
17 changed files with 1729 additions and 391 deletions
+77
View File
@@ -0,0 +1,77 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be
// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node
// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before
// the machine is read or touched, with a report naming what was refused and what is held, and the
// kept declaration left as it was.
func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) {
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
_, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
target := filepath.Join(dir, "a.conf")
declared := func(epoch, sequence int64) store.Declared {
body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence,
"resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}})
if err != nil {
t.Fatal(err)
}
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
}
held := declared(57, 3)
if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil {
t.Fatal(err)
}
stale := declared(41, 12)
report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil)
if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) ||
report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) ||
report.Refused == "" {
t.Fatalf("the refusal does not name what was refused and what is held: %+v", report)
}
if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) {
t.Fatal("the refused declaration touched the machine")
}
kept, err := store.ReadDeclared(store.DeclaredPath(opts.state))
if err != nil || string(kept.Declaration) != string(held.Declaration) {
t.Fatal("the refused declaration replaced what this node kept")
}
}
// The queue's counts are kept beside the state and read back by the next host; unreadable is an
// error, never zero.
func TestTheNumbersSurviveTheHost(t *testing.T) {
state := filepath.Join(t.TempDir(), "state.json")
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 {
t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err)
}
if err := numbersBeside(state).Save(41, 2); err != nil {
t.Fatal(err)
}
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 {
t.Fatalf("read back %d, %d, %v", sequence, refused, err)
}
if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil {
t.Fatal(err)
}
if _, _, err := numbersBeside(state).Read(); err == nil {
t.Fatal("unreadable numbers were read as zero")
}
}
+139 -74
View File
@@ -458,10 +458,10 @@ func short(digest string) string {
// them again — and everything the mesh declared read as no longer declared and removed. Even the
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, sequence int64) error {
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, order link.Order) error {
switch from {
case fromDeclared:
return refuseOlder(kept, keptErr, sequence)
return refuseOlder(kept, keptErr, order)
case fromBundle:
if known.Genesis == nil || known.Genesis.Digest == digest {
return nil
@@ -558,7 +558,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
if err := apply.CheckMode(known, d); err != nil {
return err
}
if err := refuseStale(known, kept, keptErr, digest, from, d.Sequence); err != nil {
if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil {
return err
}
@@ -1057,6 +1057,14 @@ func runLink(ctx context.Context, opts options) error {
aside, standAside := context.WithCancel(ctx)
defer standAside()
stoodAside := false
membership := link.Membership{
Node: mine.Node,
Broker: mine.Membership.Broker,
Fingerprint: mine.Membership.Fingerprint,
Password: mine.Membership.Password,
Transport: mine.Membership.Transport,
Signer: mine.Membership.Signer,
}
applier := func(ctx context.Context, raw, signature []byte) link.Report {
report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
@@ -1084,40 +1092,44 @@ func runLink(ctx context.Context, opts options) error {
}
// Two things at once, and the second is what makes disconnection ordinary. The link brings
// new declarations; this holds the machine in the last one whether the link is up or not. A
// laptop shut for a week comes back and reconciles — it does not come back and ask what it is
// (novox/hq ADR 0004).
// Reports a reconcile has to make unasked — what an adopted node holds changed, or its
// firewall did — go out over the link when it is up (novox/hq ADR 0100).
outbox := make(chan link.Unasked, 1)
// new declarations; the reconcile holds the machine in the last one whether the link is up or
// not. A laptop shut for a week comes back and reconciles — it does not come back and ask what
// it is (novox/hq ADR 0004).
//
// **Both only enqueue** (novox/hq to-be 45 §6). One worker applies, the newest declaration held
// when it starts, once, and makes one report; a reconcile due while a delivery waits is that
// delivery's apply. Reports a reconcile has to make unasked — what an adopted node holds changed,
// its firewall, its outward links — are said by the same worker, in the order they are made
// (novox/hq ADR 0100, issue 267).
watch := &adoptionWatch{}
applier = watch.noting(applier)
go holdTheMachine(ctx, opts, mine, say, sched, func(r link.Report) {
if !watch.differs(r) {
return
}
select {
case <-outbox:
// An older one nobody has published yet; this one says everything it did.
default:
}
queue := &link.Queue{
Membership: membership,
Apply: applier,
Reconcile: reconcileKept(opts, mine.Membership.Signer, sched, say),
News: func(r link.Report) bool { return worthSaying(r) && watch.differs(r) },
// Counted as said only once the broker has taken it: queued and lost — the link down, the
// publish refused — the change would never be said again (novox/hq ADR 0100).
outbox <- link.Unasked{Report: r, Done: func(published bool) {
if published {
watch.said(r)
}
}}
})
Heard: watch.said,
Unsaid: unsaidBeside(opts.state),
Numbers: numbersBeside(opts.state),
Say: say,
Timeout: opts.timeout,
}
worker := make(chan struct{})
go func() {
defer close(worker)
queue.Run(aside)
}()
// **A host starting is a reason to reconcile** (to-be 45 §6: the self-update hand-over is one of
// the four): its scheduled steps are armed from the declaration the node kept by the first apply,
// and a successor that waited five minutes for it left them unarmed for five.
queue.ReconcileDue()
go holdTheMachine(ctx, queue)
held := link.HoldRoused(aside, link.Membership{
Node: mine.Node,
Broker: mine.Membership.Broker,
Fingerprint: mine.Membership.Fingerprint,
Password: mine.Membership.Password,
Transport: mine.Membership.Transport,
Signer: mine.Membership.Signer,
}, applier, say, opts.timeout, rousedBySignal(ctx), outbox, unsaidBeside(opts.state))
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
// The act in hand finishes and is kept before this process exits: an apply that stood aside with
// no link open has its report only in the unsaid store, and only once the worker has written it.
<-worker
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
// before it has run once. The context this returns on was cancelled deliberately, so its error
// is not a fault to report.
@@ -1169,6 +1181,23 @@ func (u unsaidFile) Pending() (link.Report, bool, error) {
return r, r.Declared != "", nil
}
// numbersFile keeps the queue's report sequence and its count of declarations refused as older beside
// the node's state (novox/hq to-be 45 §6), where a successor reads them and goes on from there.
type numbersFile struct{ path string }
func numbersBeside(statePath string) numbersFile {
return numbersFile{path: store.NumbersPath(statePath)}
}
func (n numbersFile) Read() (int64, int64, error) {
kept, err := store.ReadNumbers(n.path)
return kept.ReportSequence, kept.RefusedOlder, err
}
func (n numbersFile) Save(reportSequence, refusedOlder int64) error {
return store.SaveNumbers(n.path, store.Numbers{ReportSequence: reportSequence, RefusedOlder: refusedOlder})
}
// adoptionWatch remembers what the node last said about what it holds and its firewall, so a
// reconcile speaks unasked only when that changed.
type adoptionWatch struct {
@@ -1232,15 +1261,6 @@ func (w *adoptionWatch) changed(r link.Report) bool {
return true
}
// noting wraps the applier, so a report the link publishes after a delivery counts as said.
func (w *adoptionWatch) noting(apply link.Applier) link.Applier {
return func(ctx context.Context, raw, signature []byte) link.Report {
r := apply(ctx, raw, signature)
w.changed(r)
return r
}
}
// rousedBySignal is the machine telling this process that its link is probably stale.
//
// **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would
@@ -1285,43 +1305,44 @@ func rousedBySignal(ctx context.Context) link.Roused {
// changed it, and then for ever.
const ReconcileEvery = 5 * time.Minute
func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, say link.Announce,
sched *apply.Scheduler, publish func(link.Report)) {
// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq
// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is
// waiting is that delivery's apply.
func holdTheMachine(ctx context.Context, queue *link.Queue) {
ticker := time.NewTicker(ReconcileEvery)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
queue.ReconcileDue()
}
}
}
report, err := reapplyKept(ctx, opts, mine.Membership.Signer, sched, say)
// reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it
// was last told, read once it is this act's turn, and say what did not go.
//
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or its firewall
// changed, because that is how a predecessor still writing is caught (novox/hq ADR 0100); and on any
// node when it can say which links face outside (ADR 0140) — a converged node holds nothing and found
// no firewall, so without that it could speak only in reply to a declaration, while the mesh composes
// no declaration for a node that has not said which links face outside. Whether a report is news is
// the queue's News (worthSaying and the watch), so an unchanged answer costs one comparison every
// reconcile and nothing on the bus.
func reconcileKept(opts options, signer ed25519.PublicKey, sched *apply.Scheduler,
say link.Announce) link.Reconcile {
return func(ctx context.Context) (link.Report, bool) {
report, err := reapplyKept(ctx, opts, signer, sched, say)
if errors.Is(err, store.ErrNothingDeclared) {
// Nothing to hold this machine to yet. Ordinary on a node that has enrolled and not
// been assigned anything.
continue
return link.Report{}, false
}
if err != nil {
say("cannot re-apply what this node was told: " + err.Error())
continue
}
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
// its firewall changed, because that is how a predecessor still writing is caught
// (novox/hq ADR 0100); publish decides whether anything did.
//
// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
// converged node holds nothing and found no firewall, so this gate closed on it and the
// node could speak only in reply to a declaration — while the mesh composes no declaration
// for a node that has not said which links face outside. A machine waiting for a push that
// was waiting for the machine, and measured: three converged machines sat silent while the
// control plane refused to send them a filter.
//
// Offered, not published: whether it is news is still the watch's to decide, so an
// unchanged answer costs one comparison every reconcile and nothing on the bus.
if publish != nil && worthSaying(report) {
publish(report)
return link.Report{}, false
}
switch {
case report.Refused != "":
@@ -1329,6 +1350,7 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
case len(report.Failed) > 0:
say(fmt.Sprintf("holding this machine: %d applied, and %v", len(report.Applied), report.Failed))
}
return report, true
}
}
@@ -1367,13 +1389,13 @@ func announceOr(say link.Announce) func(string) {
// applying serialises applies within this process.
//
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
// one that saves last writes a state read before the other acted — losing what the first recorded:
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
// and its record another, which is the fault every read-back in this package exists to prevent.
// Across processes — `reconcile` run by hand beside this service — the lock beside the state does
// the same (store.Lock).
// **One thing applies here: the apply queue's worker** (novox/hq to-be 45 §6), for a delivery and for
// the reconcile alike. This lock was the only order between two that applied — the link and the
// reconcile loop — and each order it allowed was met as a fault (issues 257, 261, 267); the queue is
// the order now. It stays as the guard for what may be added beside the worker: two applies
// interleaved lose what one of them recorded — a hold, the firewall found here, a resource just
// applied — and the machine is then one thing and its record another. Across processes — `reconcile`
// run by hand beside this service — the lock beside the state does the same (store.Lock).
var applying sync.Mutex
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
@@ -1418,6 +1440,21 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
if err != nil {
return link.Report{Refused: err.Error()}
}
order := link.Order{Epoch: declared.Epoch, Sequence: declared.Sequence}
// **Older than what this node applied is refused, before anything is read from the machine**
// (novox/hq to-be 45 §6, ADR 0227 rule 2): a controller that lost its lease and goes on sending.
// Read under the lock, so what it is compared with is what the last apply kept. Only a delivery:
// what a reconcile applies is what was kept.
if signed != nil {
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
if held, older := olderThanKept(kept, keptErr, order); older {
return link.Report{Declared: digestOf(raw), Order: order, OlderThan: &held,
Refused: fmt.Sprintf("this declaration was sent under lease epoch %d, sequence %d, and this "+
"node has applied epoch %d, sequence %d — from the controller holding the lease now. "+
"Refused whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)}
}
}
known, err := store.Load(opts.state)
if err != nil {
@@ -1490,7 +1527,7 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
sched.Sync(declared, held)
}
report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: runningVersion(),
report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(),
Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
// Which of this machine's links face outside, for the filter the mesh writes around them
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
@@ -1720,12 +1757,23 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
// sent, and one kept with no sequence is one this host received before it understood them; in either
// case there is no order to compare, and refusing on a guess would strand the node the moment the
// controller is older than the host. Equal is the same declaration again, which reconciling is for.
func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
//
// **And by the lease epoch first** (novox/hq to-be 45 §6): a declaration sent under an older epoch than
// the one kept here is a controller that lost its lease, whatever its sequence — the same rule the
// link's deliveries are held to (olderThanKept). Where both claim an epoch it alone decides, because a
// new lease holder's sequence says nothing against the last one's.
func refuseOlder(kept store.Declared, keptErr error, order link.Order) error {
if held, older := olderThanKept(kept, keptErr, order); older {
return fmt.Errorf("this declaration was sent under lease epoch %d, sequence %d, and the one kept "+
"here under epoch %d, sequence %d: a controller that no longer holds the lease sent it. Refused "+
"whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)
}
sequence := order.Sequence
if sequence == 0 || keptErr != nil {
return nil
}
last, err := declaration.ParseTrusted(kept.Declaration)
if err != nil || last.Sequence == 0 {
if err != nil || last.Sequence == 0 || (order.Epoch > 0 && last.Epoch > 0) {
return nil
}
if sequence < last.Sequence {
@@ -1737,6 +1785,23 @@ func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
return nil
}
// olderThanKept says whether a declaration of this order is older than the one this node kept — the
// last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept
// declaration that cannot be read claims no order: what it is compared with is unknown, and refusing
// on a guess would hold the machine off everything the mesh sends until a person looked. That kept
// file is read, and refused by name, by the next reconcile.
func olderThanKept(kept store.Declared, keptErr error, order link.Order) (link.Order, bool) {
if keptErr != nil || order.Epoch <= 0 {
return link.Order{}, false
}
last, err := declaration.ParseTrusted(kept.Declaration)
if err != nil {
return link.Order{}, false
}
held := link.Order{Epoch: last.Epoch, Sequence: last.Sequence}
return held, order.Older(held)
}
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
// report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161).
func profileAsReported(detected profile.Profile) map[string]any {
+6 -4
View File
@@ -189,13 +189,15 @@ func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
}
}
func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) {
// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue
// tells the watch every report the broker took (link.Queue.Heard).
func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) {
w := &adoptionWatch{}
report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
applier := w.noting(func(context.Context, []byte, []byte) link.Report { return report })
applier(context.Background(), nil, nil)
heard := w.said
heard(report)
if w.changed(report) {
t.Error("a reconcile repeated what the link had just published")
t.Error("a reconcile repeated what the mesh had just heard")
}
}
+44 -7
View File
@@ -5,6 +5,7 @@ import (
"strings"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
@@ -13,9 +14,15 @@ import (
// "older" could not.
func keptWith(t *testing.T, sequence int64) store.Declared {
t.Helper()
return keptAt(t, link.Order{Sequence: sequence})
}
func keptAt(t *testing.T, order link.Order) store.Declared {
t.Helper()
body, err := json.Marshal(map[string]any{
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": sequence,
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence,
"epoch": order.Epoch,
})
if err != nil {
t.Fatal(err)
@@ -24,7 +31,7 @@ func keptWith(t *testing.T, sequence int64) store.Declared {
}
func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
err := refuseOlder(keptWith(t, 7), nil, 5)
err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5})
if err == nil {
t.Fatal("sequence 5 was accepted over a kept 7")
}
@@ -34,11 +41,11 @@ func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
}
func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
if err := refuseOlder(keptWith(t, 7), nil, 8); err != nil {
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil {
t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
}
// Equal is the same declaration again, which reconciling is for.
if err := refuseOlder(keptWith(t, 7), nil, 7); err != nil {
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil {
t.Fatalf("the same sequence was refused: %v", err)
}
}
@@ -46,13 +53,43 @@ func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
// An older controller sends none; a host that received before it understood them kept none.
// Refusing on a guess would strand a node the moment the controller is older than the host.
if err := refuseOlder(keptWith(t, 7), nil, 0); err != nil {
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil {
t.Fatalf("a declaration claiming no order was refused: %v", err)
}
if err := refuseOlder(keptWith(t, 0), nil, 3); err != nil {
if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil {
t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
}
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, 3); err != nil {
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil {
t.Fatalf("a first declaration was refused: %v", err)
}
}
// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost
// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence.
func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) {
kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3})
err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12})
if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") {
t.Fatalf("an older epoch was not refused by name: %v", err)
}
if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil {
t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err)
}
if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil {
t.Fatal("a lower sequence under the same epoch was accepted")
}
// A controller with no lease — older, or rolled back to a build from before it — is today's
// behaviour: no epoch compared.
if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil {
t.Fatalf("a declaration with no epoch was refused for one: %v", err)
}
if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older ||
held != (link.Order{Epoch: 57, Sequence: 3}) {
t.Fatalf("olderThanKept said %v, holding %+v", older, held)
}
// What is kept cannot be read: no order to compare, so nothing is refused on a guess.
if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil,
link.Order{Epoch: 41, Sequence: 1}); older {
t.Fatal("refused against a kept declaration nobody can read")
}
}