novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

The node-engine hears no declaration: the queue, orders, put-backs, unsaid, numbers and news go; a session says the facts in the node-engine's own order, and the first have proves a build (R6-A A7, A8)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 16:25:11 +02:00
parent e2b6f37168
commit 1598156752
47 changed files with 739 additions and 4575 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/busy_test.go
+9 -41
View File
@@ -5,62 +5,30 @@ import (
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-host/internal/link"
)
// A container waiting because it said busy is news when it starts waiting and when it is replaced, never on
// its output alone (novox/hq ADR 0335): the walk waiting on this machine starts its gate at the report that
// says it replaced, and a reconcile is otherwise silent.
func TestABusyWaitIsSaidWhenItStartsAndWhenItEnds(t *testing.T) {
quiet := link.Report{Outward: []string{"eth0"}}
waiting := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
Container: "mesh-build-agent", State: "waiting", Said: "building a"}}}
still := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
Container: "mesh-build-agent", State: "waiting", Said: "building b, a minute on"}}}
replaced := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
Container: "mesh-build-agent", State: "replaced-when-idle"}}}
if adoptionFingerprint(waiting) == adoptionFingerprint(quiet) {
t.Error("a container starting to wait is not news")
}
if adoptionFingerprint(still) != adoptionFingerprint(waiting) {
t.Error("a wait whose output changed reads as news on every ask")
}
if adoptionFingerprint(replaced) == adoptionFingerprint(waiting) {
t.Error("a container replaced after it waited is not news")
}
if !worthSaying(link.Report{Busy: replaced.Busy}) {
t.Error("a replacement after a wait is not worth saying")
}
}
// While a container waits, a pass is asked for every so often, not at the reconcile's five minutes; with
// nothing waiting, none (novox/hq ADR 0335 §2).
func TestAWaitIsAskedAgainSoonerThanTheReconcile(t *testing.T) {
if BusyAskEvery >= ReconcileEvery || BusyAskEvery > time.Minute {
// While a container waits, a pass is asked for every so often, not at some long interval; with nothing waiting, none
// (novox/hq ADR 0335 §2, ADR 0360 §11).
func TestAWaitIsAskedAgainSoon(t *testing.T) {
if BusyAskEvery > time.Minute {
t.Fatalf("a waiting container is asked every %s", BusyAskEvery)
}
var reconciles atomic.Int32
queue := &link.Queue{Reconcile: func(context.Context) (link.Report, bool) {
reconciles.Add(1)
return link.Report{}, false
}}
var asked atomic.Int32
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go queue.Run(ctx)
busyWaiting.Store(false)
t.Cleanup(func() { busyWaiting.Store(false) })
go busyPasses(ctx, queue, 10*time.Millisecond)
go busyPasses(ctx, func() { asked.Add(1) }, 10*time.Millisecond)
time.Sleep(100 * time.Millisecond)
if n := reconciles.Load(); n != 0 {
if n := asked.Load(); n != 0 {
t.Fatalf("%d passes were asked with nothing waiting", n)
}
busyWaiting.Store(true)
deadline := time.Now().Add(2 * time.Second)
for reconciles.Load() == 0 && time.Now().Before(deadline) {
for asked.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if reconciles.Load() == 0 {
if asked.Load() == 0 {
t.Fatal("no pass was asked while a container waits")
}
}
cmd/mesh-host/epoch_test.go
-77
View File
@@ -1,77 +0,0 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be
// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node
// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before
// the machine is read or touched, with a report naming what was refused and what is held, and the
// kept declaration left as it was.
func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) {
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
_, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
target := filepath.Join(dir, "a.conf")
declared := func(epoch, sequence int64) store.Declared {
body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence,
"resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}})
if err != nil {
t.Fatal(err)
}
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
}
held := declared(57, 3)
if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil {
t.Fatal(err)
}
stale := declared(41, 12)
report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil)
if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) ||
report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) ||
report.Refused == "" {
t.Fatalf("the refusal does not name what was refused and what is held: %+v", report)
}
if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) {
t.Fatal("the refused declaration touched the machine")
}
kept, err := store.ReadDeclared(store.DeclaredPath(opts.state))
if err != nil || string(kept.Declaration) != string(held.Declaration) {
t.Fatal("the refused declaration replaced what this node kept")
}
}
// The queue's counts are kept beside the state and read back by the next host; unreadable is an
// error, never zero.
func TestTheNumbersSurviveTheHost(t *testing.T) {
state := filepath.Join(t.TempDir(), "state.json")
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 {
t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err)
}
if err := numbersBeside(state).Save(41, 2); err != nil {
t.Fatal(err)
}
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 {
t.Fatalf("read back %d, %d, %v", sequence, refused, err)
}
if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil {
t.Fatal(err)
}
if _, _, err := numbersBeside(state).Read(); err == nil {
t.Fatal("unreadable numbers were read as zero")
}
}
cmd/mesh-host/generation_floor_test.go
-40
View File
@@ -1,40 +0,0 @@
package main
import (
"context"
"testing"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// **The generation applied is kept, and a put-back never lowers it**: after a put-back of generation 5
// over 7, a send from generation 6 is still older than what this machine applied.
func TestThePutBackDoesNotLowerTheGenerationApplied(t *testing.T) {
built, err := system.For("arch")
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
t.Skip("this machine is not one these tests can apply on")
}
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
f := machineThatApplied(t, 7)
newer := f.declared(t, map[string]any{"sequence": 9, "generation": 8}, "docker.conf", f.stays)
if r := applyAndKeep(context.Background(), f.opts, newer.Declaration, &newer, nil, nil); r.Refused != "" {
t.Fatalf("a newer generation was refused: %s", r.Refused)
}
back := f.declared(t, map[string]any{"sequence": 10, "generation": 5, "put_back": true}, "docker.conf", f.stays)
if r := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil); r.Refused != "" {
t.Fatalf("a put-back was refused: %s", r.Refused)
}
state, err := store.Load(f.opts.state)
if err != nil || state.Generation != 8 {
t.Fatalf("the generation applied is %d after a put-back of 5 over 8 (%v)", state.Generation, err)
}
stale := f.declared(t, map[string]any{"sequence": 11, "generation": 7}, "docker.conf", f.stays)
if r := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil); r.OlderGeneration == nil {
t.Fatalf("a send from generation 7 after 8 was applied because a put-back came between: %+v", r)
}
}
cmd/mesh-host/generation_test.go
-170
View File
@@ -1,170 +0,0 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// The assignment generation a declaration was composed from (novox/hq issue 234).
//
// On 2026-10-04 a declaration newer in sequence than every other, and composed from a view of the
// assignments older than the one this machine had applied, named four fewer modules — and the host,
// ordering by sequence alone, undeclared all four. The sequence orders arrival; it cannot tell a later
// send carrying earlier content. The generation can: the node-engine refuses a declaration composed from
// an older generation than the one it applied, before anything is read from the machine.
//
// These read what the report says through its JSON, as the controller does, so they run against a
// node-engine that has never heard of a generation and fail there for that reason.
// generationFixture is a machine that applied generation 7: a file the mesh declared, kept in its state,
// and the declaration it kept — signed — that named it.
type generationFixture struct {
opts options
private ed25519.PrivateKey
kept store.Declared
// stays is the file the older generation's declaration does not name.
stays string
}
func machineThatApplied(t *testing.T, generation int64) generationFixture {
t.Helper()
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
_, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
stays := filepath.Join(dir, "docker.conf")
if err := os.WriteFile(stays, []byte("kept\n"), 0o644); err != nil {
t.Fatal(err)
}
// The state as a node-engine that applied generation 7 keeps it: the file it owns, and the highest
// generation it applied. Written as JSON, so the fixture is the same bytes for any build under test.
state := map[string]any{"resources": []any{map[string]any{"id": "docker.conf", "type": "file",
"origin": store.OriginDeclared, "target": stays}}, "generation": generation}
raw, err := json.Marshal(state)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(opts.state, raw, 0o600); err != nil {
t.Fatal(err)
}
f := generationFixture{opts: opts, private: private, stays: stays}
f.kept = f.declared(t, map[string]any{"sequence": 3, "generation": generation}, "docker.conf", stays)
if err := store.SaveDeclared(store.DeclaredPath(opts.state), f.kept); err != nil {
t.Fatal(err)
}
return f
}
// declared is a signed declaration with the envelope's keys given and one file.
func (f generationFixture) declared(t *testing.T, envelope map[string]any, id, path string) store.Declared {
t.Helper()
body := map[string]any{"declaration": 1,
"resources": []any{map[string]any{"id": id, "type": "file", "path": path, "content": "x\n"}}}
for k, v := range envelope {
body[k] = v
}
raw, err := json.Marshal(body)
if err != nil {
t.Fatal(err)
}
return store.Declared{Declaration: raw, Signature: ed25519.Sign(f.private, raw)}
}
// reportJSON is a report as the controller reads it.
func reportJSON(t *testing.T, r link.Report) map[string]any {
t.Helper()
raw, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
t.Fatal(err)
}
return out
}
// **A newer sequence from an older generation is refused, and nothing is undeclared** (issue 234).
func TestANewerSequenceFromAnOlderGenerationIsRefusedAndNothingIsUndeclared(t *testing.T) {
f := machineThatApplied(t, 7)
other := filepath.Join(filepath.Dir(f.opts.state), "sudo.conf")
stale := f.declared(t, map[string]any{"sequence": 9, "generation": 6}, "sudo.conf", other)
report := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil)
said := reportJSON(t, report)
older, _ := said["older_generation"].(map[string]any)
if older == nil || older["generation"] != float64(6) || older["applied"] != float64(7) {
t.Fatalf("the refusal does not name the generation refused and the one applied: %v", said)
}
if report.Refused == "" || !strings.Contains(report.Refused, "generation 6") ||
!strings.Contains(report.Refused, "generation 7") {
t.Errorf("the refusal is not said in words naming both generations: %q", report.Refused)
}
if report.Declared != digestOf(stale.Declaration) || report.Sequence != 9 {
t.Errorf("the refusal does not name the declaration refused, so the mesh cannot name who sent it: %+v", report)
}
if _, err := os.Stat(f.stays); err != nil {
t.Fatalf("a refused declaration undeclared what the machine applied: %v", err)
}
if _, err := os.Stat(other); err == nil {
t.Error("the refused declaration touched the machine")
}
state, err := store.Load(f.opts.state)
if err != nil {
t.Fatal(err)
}
if _, owned := state.Find("docker.conf"); !owned {
t.Error("the refused declaration took a resource out of what this machine owns")
}
kept, err := store.ReadDeclared(store.DeclaredPath(f.opts.state))
if err != nil || string(kept.Declaration) != string(f.kept.Declaration) {
t.Error("the refused declaration replaced what this node kept, so the next reconcile would apply it")
}
}
// refusedForItsGeneration says a report refused its declaration for the generation it came from.
func refusedForItsGeneration(t *testing.T, r link.Report) bool {
t.Helper()
return reportJSON(t, r)["older_generation"] != nil || strings.Contains(r.Refused, "generation")
}
// refusedAsUnread says a report refused its declaration as one this node-engine cannot read.
func refusedAsUnread(r link.Report) bool { return strings.Contains(r.Refused, "not a declaration") }
// **A gate's put-back is applied, whatever generation it carries**: it carries the generation of what it
// puts back, and says it is a put-back (issue 234).
func TestAPutBackFromAnOlderGenerationIsNotRefused(t *testing.T) {
f := machineThatApplied(t, 7)
back := f.declared(t, map[string]any{"sequence": 9, "generation": 5, "put_back": true}, "docker.conf", f.stays)
report := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil)
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
t.Fatalf("a put-back was refused: %s", report.Refused)
}
}
// **A declaration that carries no generation is applied as it always was**: an older controller sends
// none, and a node-engine that refused it would strand every machine the moment the controller is older.
func TestADeclarationWithoutAGenerationIsNotRefusedForOne(t *testing.T) {
f := machineThatApplied(t, 7)
older := f.declared(t, map[string]any{"sequence": 9}, "docker.conf", f.stays)
report := applyAndKeep(context.Background(), f.opts, older.Declaration, &older, nil, nil)
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
t.Fatalf("a declaration without a generation was refused: %s", report.Refused)
}
// And the same generation again is no older.
same := f.declared(t, map[string]any{"sequence": 10, "generation": 7}, "docker.conf", f.stays)
report = applyAndKeep(context.Background(), f.opts, same.Declaration, &same, nil, nil)
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
t.Fatalf("a declaration of the generation applied was refused: %s", report.Refused)
}
}
cmd/mesh-host/local_network_test.go
-29
View File
@@ -1,29 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/outward"
"github.com/novox/mesh-host/internal/profile"
)
func reportOn(networks ...outward.Network) link.Report {
return link.Report{Outward: []string{"eth0"},
Profile: profileAsReported(profile.Profile{Machine: &profile.Machine{Networks: networks}})}
}
// **A change of the networks a machine is attached to is said unasked** (novox/hq ADR 0340 §7): a port
// reached from the local network is admitted from them, so a laptop that joins another network is said at
// the next reconcile, not at the next push — and the same networks again are not news.
func TestAChangedNetworkIsSaidUnasked(t *testing.T) {
w := &adoptionWatch{}
home := reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})
w.said(home)
if w.differs(reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})) {
t.Fatal("the same networks read as a change, so the machine would speak on every reconcile")
}
if !w.differs(reportOn(outward.Network{Link: "eth0", Network: "10.20.0.0/16"})) {
t.Fatal("a machine that moved to another network does not say so until it is pushed to")
}
}
cmd/mesh-host/machine_refresh_test.go
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/link"
)
// Defends novox/hq issue 160: a machine nobody pushed to still says what it is made of once a day, so the
// newest of it the mesh holds is no older than a day. Before, the reconcile said it only with something
// else (a hold, a firewall, an outward link changed), and a converged machine's picture was as old as the
// last delivery to it.
func TestAMachineNobodyPushedToSaysWhatItIsOnceADay(t *testing.T) {
at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
w := &adoptionWatch{now: func() time.Time { return at }}
quiet := link.Report{Outward: []string{"wlan0"},
Profile: map[string]any{"machine": map[string]any{"observed_at": at}}}
if !w.news(quiet) {
t.Fatal("the first reconcile after the engine started did not say what the machine is")
}
w.said(quiet)
for _, after := range []time.Duration{5 * time.Minute, 12 * time.Hour, 22*time.Hour + 55*time.Minute} {
at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(after)
if w.news(quiet) {
t.Errorf("%s after the mesh heard it, an unchanged reconcile was said again", after)
}
}
// The reconcile that falls past 23 hours says it, so it reaches the mesh inside the day.
at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(23*time.Hour + 5*time.Minute)
if !w.news(quiet) {
t.Fatal("a day on, the reconcile did not say what the machine is")
}
w.said(quiet)
at = at.Add(5 * time.Minute)
if w.news(quiet) {
t.Error("once said, it was said again at the next reconcile")
}
}
// A delivery's account carries the profile too: heard, it starts the day again.
func TestADeliveryHeardStartsTheDayAgain(t *testing.T) {
at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
w := &adoptionWatch{now: func() time.Time { return at }}
report := link.Report{Outward: []string{"eth0"}, Profile: map[string]any{"capabilities": []any{}}}
w.said(report)
at = at.Add(20 * time.Hour)
w.said(report) // a delivery's apply
at = at.Add(20 * time.Hour)
if w.news(report) {
t.Error("said again unasked 20 hours after a delivery told the mesh")
}
}
// A refusal carries nothing about the machine, so it is never said for being a day old.
func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) {
at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
w := &adoptionWatch{now: func() time.Time { return at }}
if w.machineDue(link.Report{Refused: "no"}) {
t.Error("a refusal was due for what the machine is made of")
}
}
// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic
// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses
// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first
// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no
// monotonic reading, and judge the day as the wall clock says it.
func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) {
before := time.Now() // carries a monotonic reading
asleep := before.Add(3 * time.Second)
if asleep.Sub(before) != 3*time.Second {
t.Fatal("the monotonic reading is not what this test simulates with")
}
calls := 0
w := &adoptionWatch{now: func() time.Time {
calls++
if calls == 1 {
return before
}
// The wall clock a day on, as a machine's clock reads after waking.
return asleep.Round(0).Add(24 * time.Hour)
}}
report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}}
w.said(report)
if strings.Contains(w.machineSaid.String(), "m=") {
t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid)
}
if !w.news(report) {
t.Error("a day of wall time on, a laptop that slept did not say what it is")
}
}
// And the watch's own clock never carries a monotonic reading.
func TestTheWatchClockIsWallTime(t *testing.T) {
w := &adoptionWatch{}
if s := w.clock().String(); strings.Contains(s, "m=") {
t.Errorf("the watch's clock carries a monotonic reading: %s", s)
}
w = &adoptionWatch{now: time.Now}
if s := w.clock().String(); strings.Contains(s, "m=") {
t.Errorf("a given clock's time kept its monotonic reading: %s", s)
}
}
cmd/mesh-host/main.go
+92 -612
View File
@@ -105,8 +105,6 @@ reach, run as root at the machine.
profile what this machine can be asked to do
inventory what this machine is, and what it holds
apply FILE make this machine match a declaration from a file
reconcile make this machine match what the mesh last told it — or, before any
mesh has, the bundle this host carries
bundle show what this host carries
key make this machine's tunnel key, or read the one it made, and print the public
half: what its join token is issued for (novox/hq ADR 0169)
@@ -126,10 +124,9 @@ reach, run as root at the machine.
--state where this node keeps what it knows (default /var/lib/mesh-host/state.json)
--dry-run say what applying would change, and change nothing
Both apply and reconcile say what they would change before changing anything, and refuse a
declaration for the other mode than this node is in, or one older than what the mesh last said.
It connects to nothing and listens on nothing. What it applies comes from a file.
Apply says what it would change before changing anything, and refuses a declaration for the other mode than this
node is in. It is for a machine the mesh has not spoken to. Once enrolled, the node follows its wants (novox/hq ADR
0539): it applies a module from the composition its want names, and no whole declaration.
`
func main() {
@@ -286,13 +283,6 @@ func run(ctx context.Context, command string, opts options) error {
}
return runApply(ctx, opts, d, raw, fromFile)
case "reconcile":
d, raw, from, err := reconcileSource(opts)
if err != nil {
return err
}
return runApply(ctx, opts, d, raw, from)
case "bundle":
if bundle.IsEmpty(builtFor) {
fmt.Println("this host carries no bundle")
@@ -412,58 +402,12 @@ type provenance int
const (
// fromFile is `apply FILE`: handed to the host by someone already running it as root.
fromFile provenance = iota
// fromBundle is `reconcile` on a machine the mesh has told nothing yet: the bundle carried in
// fromBundle is genesis on a machine the mesh has told nothing yet: the bundle carried in
// the binary, the first node's path before its mesh is up (novox/hq ADR 0004).
fromBundle
// fromDeclared is `reconcile` on a node the mesh has spoken to: what it last said, kept
// signed beside the state (declared.json), verified again before it is applied.
fromDeclared
)
// reconcileSource is which declaration `reconcile` holds this machine to.
//
// **What the mesh last said, never the bundle, once the mesh has said anything** (novox/hq issue
// 104). The bundle is right at genesis and stale a minute later — genesis rewrites it for the
// machine before applying it, and every declaration since came from the controller — and on an
// adopted node it is a converged declaration for a machine that is not converged. A node that
// has been told something and cannot prove it is the mesh's is refused, with the reason; the
// bundle is not applied in its place.
func reconcileSource(opts options) (*declaration.Declaration, []byte, provenance, error) {
path := store.DeclaredPath(opts.state)
_, err := store.ReadDeclared(path)
switch {
case err == nil:
mine, err := identity.Load(identity.Path(opts.state))
if err != nil {
return nil, nil, 0, fmt.Errorf("this node was told a declaration by the mesh, kept at %s, "+
"and cannot prove it is the mesh's: %w\n\nIt is not applied unproven, and the bundle "+
"this host carries is not applied in its place: that was consumed at genesis", path, err)
}
raw, err := store.LoadDeclared(path, mine.Membership.Signer)
if err != nil {
return nil, nil, 0, fmt.Errorf("%w\n\nThe bundle this host carries is not applied in its "+
"place: that was consumed at genesis", err)
}
d, err := declaration.Parse(raw)
if err != nil {
return nil, nil, 0, err
}
return d, raw, fromDeclared, nil
case errors.Is(err, store.ErrNothingDeclared):
d, err := bundle.Load(builtFor)
if err != nil {
return nil, nil, 0, err
}
return d, bundle.Raw(builtFor), fromBundle, nil
default:
return nil, nil, 0, err
}
}
func (p provenance) origin() string {
if p == fromDeclared {
return store.OriginDeclared
}
// A file handed to the host is applied as the bundle is: what it puts here is invisible to
// the removal pass of a declaration that later arrives from the mesh (novox/hq issue 010).
return store.OriginCarried
@@ -473,8 +417,6 @@ func (p provenance) name(opts options) string {
switch p {
case fromBundle:
return "the carried bundle"
case fromDeclared:
return "what the mesh last told this node (" + store.DeclaredName + ")"
}
return opts.file
}
@@ -499,10 +441,8 @@ func short(digest string) string {
// them again — and everything the mesh declared read as no longer declared and removed. Even the
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, order link.Order) error {
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
switch from {
case fromDeclared:
return refuseOlder(kept, keptErr, order)
case fromBundle:
if known.Genesis == nil || known.Genesis.Digest == digest {
return nil
@@ -536,8 +476,8 @@ func refuseStale(known store.State, kept store.Declared, keptErr error, digest s
return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+
"node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+
"recorded as this machine's own, which the mesh could then never remove, and what the mesh "+
"declared read as no longer declared — so no file is applied here: `reconcile` applies what "+
"the mesh last said, and `push` from the controller changes it",
"declared read as no longer declared — so no file is applied here: the node follows its wants "+
"(novox/hq ADR 0539)",
short(last), store.DeclaredName, what)
}
@@ -585,13 +525,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
// declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the
// note, or to the kept declaration when the note predates it.
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
if from == fromDeclared {
if known.Mode != "" && known.Mode != apply.ModeOf(d) {
fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n",
known.Mode, apply.ModeOf(d))
}
known.Mode = apply.ModeOf(d)
} else if known.Mode == "" && keptErr == nil {
if known.Mode == "" && keptErr == nil {
if last, err := declaration.Parse(kept.Declaration); err == nil {
known.Mode = apply.ModeOf(last)
}
@@ -599,14 +533,9 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
if err := apply.CheckMode(known, d); err != nil {
return err
}
if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil {
if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
return err
}
// The generation, as the link's deliveries are held to it (novox/hq issue 234): one composed from an
// older view of the assignments than this machine applied would undeclare what the mesh still assigns.
if refused := olderGeneration(known, d); refused != nil {
return errors.New(olderGenerationWords(*refused))
}
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
@@ -674,7 +603,6 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
// What this apply settles about the node, whichever way it went. The mode is what the
// declaration said and the check above agreed with; the bundle, once applied, is consumed.
updated.Mode = apply.ModeOf(d)
updated.Generation = appliedGeneration(known, d) // raised, never lowered (novox/hq issue 234)
if from == fromBundle {
updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()}
}
@@ -1179,8 +1107,9 @@ func runLink(ctx context.Context, opts options) error {
Signer: mine.Membership.Signer,
}
applier := func(ctx context.Context, raw, signature []byte) link.Report {
report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
// What follows an apply of a module (the node agent's, novox/hq ADR 0539): a membership it delivered for another bus
// is adopted, and a newer node-engine it placed makes this one stand aside.
afterApply := func() {
// **A declaration may carry this machine's membership for another bus.** It arrives as a
// sealed file like any secret, and is read after the rest has applied so the bus it names is
// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
@@ -1201,35 +1130,16 @@ func runLink(ctx context.Context, opts options) error {
stoodAside = true
standAside()
}
return report
}
// Two things at once, and the second is what makes disconnection ordinary. The link brings
// new declarations; the reconcile holds the machine in the last one whether the link is up or
// not. A laptop shut for a week comes back and reconciles — it does not come back and ask what
// it is (novox/hq ADR 0004).
//
// **Both only enqueue** (novox/hq to-be 45 §6). One worker applies, the newest declaration held
// when it starts, once, and makes one report; a reconcile due while a delivery waits is that
// delivery's apply. Reports a reconcile has to make unasked — what an adopted node holds changed,
// its firewall, its outward links — are said by the same worker, in the order they are made
// (novox/hq ADR 0100, issue 267).
watch := &adoptionWatch{}
// The node follows its wants; it is never sent a declaration (novox/hq ADR 0539 §9). What the session does is hold
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
// controller asks.
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
queue := &link.Queue{
queue := &link.Session{
Membership: membership,
Apply: applier,
Reconcile: reconcileKept(opts, mine.Membership.Signer, sched, say),
News: watch.news,
// Counted as said only once the broker has taken it: queued and lost — the link down, the
// publish refused — the change would never be said again (novox/hq ADR 0100).
// And the first account under this build is what proves it to the launcher (to-be 45 §8).
Heard: func(r link.Report) {
watch.said(r)
proof.heard(r)
},
Unsaid: unsaidBeside(opts.state),
Numbers: numbersBeside(opts.state),
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
HaveWritten: proof.haveWritten,
// And the operator's hand-over of a directory used as found, carried by the controller from the
// terminal (novox/hq issue 356): judged and recorded here, applied by the next apply.
HandOver: handOverFrom(opts.state),
@@ -1248,6 +1158,10 @@ func runLink(ctx context.Context, opts options) error {
Say: say,
Timeout: opts.timeout,
}
queue.OnLink = func(ctx context.Context) {
sayFacts(ctx, opts, mine.Membership.Signer, queue, say)
afterApply()
}
// And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3,
// issue 676): the vault is asked over this queue's link, so the queue is named once it exists.
queue.MoveSecret = moveSecretFrom(opts.state, queue)
@@ -1258,25 +1172,6 @@ func runLink(ctx context.Context, opts options) error {
}, func(verb string, args any) (json.RawMessage, error) {
return queue.AskVault(ctx, verb, args)
}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo)
worker := make(chan struct{})
go func() {
defer close(worker)
queue.Run(aside)
}()
// **A host starting is a reason to reconcile** (to-be 45 §6: the self-update hand-over is one of
// the four): its scheduled steps are armed from the declaration the node kept by the first apply,
// and a successor that waited five minutes for it left them unarmed for five.
//
// **A host on trial says its account whether or not it is news** (to-be 45 §8): the launcher
// waits for this build to report its declaration, and a converged machine with nothing new to
// say would otherwise not say anything until the mesh next sent it something.
if unproved(opts.state, runningVersion()) {
queue.ReportAsked()
} else {
queue.ReconcileDue()
}
go holdTheMachine(ctx, queue)
go busyPasses(ctx, queue, BusyAskEvery)
// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
// ADR 0240) — between applies, which is when a container crash-loops.
if judging != nil {
@@ -1301,9 +1196,6 @@ func runLink(ctx context.Context, opts options) error {
go serveFilling(aside, opts, mine.Membership.Signer, say)
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
// The act in hand finishes and is kept before this process exits: an apply that stood aside with
// no link open has its report only in the unsaid store, and only once the worker has written it.
<-worker
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
// before it has run once. The context this returns on was cancelled deliberately, so its error
// is not a fault to report.
@@ -1313,241 +1205,6 @@ func runLink(ctx context.Context, opts options) error {
return held
}
// unsaidFile keeps the last apply's report beside the node's state until the mesh has taken it
// (novox/hq issue 264), so a report lost when this host stood aside for its successor — or died
// between applying and publishing — is said by whichever host runs next, once it is linked.
type unsaidFile struct{ path string }
func unsaidBeside(statePath string) unsaidFile { return unsaidFile{path: store.UnsaidPath(statePath)} }
func (u unsaidFile) Keep(r link.Report) error {
// A report naming no declaration is one the mesh cannot match to what it sent, and keeping it
// would only re-say an older apply after a newer one was refused.
if r.Declared == "" {
return store.ClearUnsaid(u.path)
}
body, err := json.Marshal(r)
if err != nil {
return err
}
return store.SaveUnsaid(u.path, body)
}
func (u unsaidFile) Said(declared string) error {
kept, ok, err := u.Pending()
if err != nil || !ok || kept.Declared != declared {
return err
}
return store.ClearUnsaid(u.path)
}
func (u unsaidFile) Pending() (link.Report, bool, error) {
raw, err := store.ReadUnsaid(u.path)
if err != nil || raw == nil {
return link.Report{}, false, err
}
var r link.Report
if err := json.Unmarshal(raw, &r); err != nil {
// Unreadable is forgotten rather than kept for ever: it can never be said.
_ = store.ClearUnsaid(u.path)
return link.Report{}, false, fmt.Errorf("the kept report at %s is unreadable, and is dropped: %w", u.path, err)
}
return r, r.Declared != "", nil
}
// numbersFile keeps the queue's report sequence and its count of declarations refused as older beside
// the node's state (novox/hq to-be 45 §6), where a successor reads them and goes on from there.
type numbersFile struct{ path string }
func numbersBeside(statePath string) numbersFile {
return numbersFile{path: store.NumbersPath(statePath)}
}
func (n numbersFile) Read() (int64, int64, error) {
kept, err := store.ReadNumbers(n.path)
return kept.ReportSequence, kept.RefusedOlder, err
}
func (n numbersFile) Save(reportSequence, refusedOlder int64) error {
return store.SaveNumbers(n.path, store.Numbers{ReportSequence: reportSequence, RefusedOlder: refusedOlder})
}
// adoptionWatch remembers what the node last said about what it holds and its firewall, so a
// reconcile speaks unasked only when that changed.
type adoptionWatch struct {
mu sync.Mutex
last string
// faulty is whether the last report the mesh heard said an apply refused or failed (novox/hq issue 233).
faulty bool
// machineSaid is when the mesh last heard what this machine is made of (novox/hq issue 160), and now the
// clock it is judged by (time.Now when nil).
machineSaid time.Time
now func() time.Time
}
// machineEvery is how old what a machine said of itself may grow before a reconcile says it again unasked
// (novox/hq issue 160): a machine nobody pushed to still says it once a day, so no fact the mesh holds of it
// is older than a day. Under a day by more than one reconcile interval, so the reconcile that says it falls
// inside the day rather than just past it.
const machineEvery = 23 * time.Hour
// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's
// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept
// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day.
func (w *adoptionWatch) clock() time.Time {
if w.now != nil {
return w.now().Round(0)
}
return time.Now().Round(0)
}
// machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago
// or more. A refusal carries none, so it never is.
func (w *adoptionWatch) machineDue(r link.Report) bool {
return r.Refused == "" && len(r.Profile) > 0 && w.clock().Sub(w.machineSaid) >= machineEvery
}
// fingerprint is what a report says about adoption: each hold and whether it changed, the
// firewall, and what is reachable on the machine — which only an adopted node reports, and which
// is what the controller previews a flip from, so a port that opens or closes between deliveries
// must reach it too (novox/hq ADR 0100).
func adoptionFingerprint(r link.Report) string {
// **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR
// 0140). The mesh composes no filter for a machine that has not said, so a machine whose links
// changed — or which has only just learnt to say — has to say so without being asked. Left out,
// it could only speak when a declaration arrived, and a declaration cannot be composed until it
// has spoken: a machine waiting for a push that is waiting for the machine.
parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
// **And the networks those links are on** (novox/hq ADR 0340 §7): a port reached from the local
// network is admitted from them, so a laptop that joins another network, or a lease that moves, is
// said at the next reconcile rather than at the next push or the next day.
if networks := attachedNetworksOf(r); networks != "" {
parts = append(parts, "networks="+networks)
}
// And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk
// waiting on this machine starts its gate at the report that says it replaced. Not its output, which
// may change on every ask.
for _, b := range r.Busy {
parts = append(parts, "busy "+b.ID+"="+b.State)
}
for _, h := range r.Held {
parts = append(parts, "held "+h.ID+"="+h.Changed)
}
// And what filters the machine, with the found firewall's state (novox/hq ADR 0168): a rule the
// operator removes between declarations, or a front end enabled again, is said at the next
// reconcile rather than at the next push.
for _, f := range r.Filters {
parts = append(parts, "filter "+f.Owner+" "+f.Where+" "+f.Refuses)
}
// And a witness's verdict (to-be 45 §8): one reached or one ended is said at the next reconcile.
for _, v := range r.Rollbacks {
parts = append(parts, fmt.Sprintf("rollback %s %s %s %s", v.Component, v.From, v.To, v.Outcome))
}
// **And how the apply went** (novox/hq issue 233): refused, clean, or which resources it did not apply. A
// reconcile is the only apply a machine makes between deliveries, so a machine whose package manager stopped
// answering, or answers again, says so at the next reconcile — otherwise the mesh's record of what it did
// stays whatever the last delivery left, and a fault that began or ended between deliveries is never seen.
// Not the words: an error that carries a time would read as new on every reconcile.
parts = append(parts, applyState(r))
if r.FoundFirewall != nil {
parts = append(parts, fmt.Sprintf("found-firewall %s active=%v retired-by=%s", r.FoundFirewall.Kind,
r.FoundFirewall.Active, r.FoundFirewall.RetiredBy))
}
for _, reach := range r.Reachable {
parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
reach.Port, reach.By, reach.Published, reach.ContainerPort))
}
sort.Strings(parts[2:])
return strings.Join(parts, "\n")
}
// attachedNetworksOf is the networks a report's profile says the machine is attached to, as one line: the
// part of what the machine is made of that the filter is composed from (novox/hq ADR 0340).
func attachedNetworksOf(r link.Report) string {
machine, ok := r.Profile["machine"].(map[string]any)
if !ok {
return ""
}
networks, ok := machine["networks"].([]any)
if !ok {
return ""
}
said := make([]string, 0, len(networks))
for _, n := range networks {
if entry, ok := n.(map[string]any); ok {
said = append(said, fmt.Sprintf("%v %v", entry["link"], entry["network"]))
}
}
return strings.Join(said, ",")
}
// applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not
// apply and which keys its failure is said under.
func applyState(r link.Report) string {
switch {
case r.Refused != "":
return "apply refused"
case len(r.Failed) == 0:
return "apply clean"
}
keys := make([]string, 0, len(r.Failed))
for k := range r.Failed {
keys = append(keys, k)
}
sort.Strings(keys)
return "apply failed " + strings.Join(keys, ",") + " unapplied " + strings.Join(r.Unapplied, ",")
}
// faulty is whether a report says an apply refused or failed.
func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 }
// news is whether a reconcile's report is worth saying unasked: what the machine is made of last reached the
// mesh a day ago or more (novox/hq issue 160); or it says something the last one the mesh heard
// did not, and it is about the machine (worthSaying) — or about how the apply went, refused or failed now, or
// clean after the last one the mesh heard was not (novox/hq issue 233). A refusal says nothing about the
// machine, but that the machine refuses is what the mesh raises a condition from, and a clean apply after it is
// what clears one.
func (w *adoptionWatch) news(r link.Report) bool {
w.mu.Lock()
defer w.mu.Unlock()
// What the machine is made of, once a day whatever else changed (novox/hq issue 160).
if w.machineDue(r) {
return true
}
if adoptionFingerprint(r) == w.last {
return false
}
return worthSaying(r) || faulty(r) || w.faulty
}
// differs says whether a report says anything the last one that went out did not. It records
// nothing: what was said is what reached the mesh, not what was written down to send.
func (w *adoptionWatch) differs(r link.Report) bool {
w.mu.Lock()
defer w.mu.Unlock()
return adoptionFingerprint(r) != w.last
}
// said records a report the mesh has actually been told.
func (w *adoptionWatch) said(r link.Report) {
w.mu.Lock()
defer w.mu.Unlock()
w.last = adoptionFingerprint(r)
w.faulty = faulty(r)
if len(r.Profile) > 0 {
w.machineSaid = w.clock()
}
}
// changed is differs and said together, for a report published as it is made.
func (w *adoptionWatch) changed(r link.Report) bool {
if !w.differs(r) {
return false
}
w.said(r)
return true
}
// rousedBySignal is the machine telling this process that its link is probably stale.
//
// **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would
@@ -1586,14 +1243,6 @@ func rousedBySignal(ctx context.Context) link.Roused {
return out
}
// ReconcileEvery is how often a node re-applies what it was last told.
//
// Not driven by the link. Changes are pushed, so this is not polling for them — it is the answer
// to a machine drifting: a file edited by hand, a container stopped by somebody, a service that
// died. A node that only acted when told would hold its state exactly until something else
// changed it, and then for ever.
const ReconcileEvery = 5 * time.Minute
// judging is the serving host's liveness judge (novox/hq ADR 0240); nil in a one-shot command, which
// reports no health, and in a test.
var judging *liveness.Judge
@@ -1690,7 +1339,7 @@ const (
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) {
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) {
ticker := time.NewTicker(liveness.LookEvery)
defer ticker.Stop()
var lastSaid time.Time
@@ -1911,22 +1560,6 @@ func orNothing(s, unless string) string {
return s
}
// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq
// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is
// waiting is that delivery's apply.
func holdTheMachine(ctx context.Context, queue *link.Queue) {
ticker := time.NewTicker(ReconcileEvery)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
queue.ReconcileDue()
}
}
}
// busyWaiting is whether the last apply left a container running because it said busy (novox/hq ADR 0335).
var busyWaiting atomic.Bool
@@ -1935,10 +1568,9 @@ var busyWaiting atomic.Bool
// within the reconcile's five minutes — and the bound of 15 minutes is met within this of its end.
const BusyAskEvery = 20 * time.Second
// busyPasses asks for a pass every so often while a container waits because it said busy. **It asks; it
// does not apply**, as holdTheMachine: the queue's worker does, when its turn comes, so a pass never holds
// the machine for the wait, and a delivery waiting is that pass's apply.
func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) {
// busyPasses asks for a pass every so often while a container waits because it said busy (ADR 0360 §11). **It asks; it
// does not apply**: the node agent does, when its turn comes, so a pass never holds the machine for the wait.
func busyPasses(ctx context.Context, ask func(), every time.Duration) {
ticker := time.NewTicker(every)
defer ticker.Stop()
for {
@@ -1947,66 +1579,12 @@ func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) {
return
case <-ticker.C:
if busyWaiting.Load() {
queue.ReconcileDue()
ask()
}
}
}
}
// reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it
// was last told, read once it is this act's turn, and say what did not go.
//
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or its firewall
// changed, because that is how a predecessor still writing is caught (novox/hq ADR 0100); and on any
// node when it can say which links face outside (ADR 0140) — a converged node holds nothing and found
// no firewall, so without that it could speak only in reply to a declaration, while the mesh composes
// no declaration for a node that has not said which links face outside. Whether a report is news is
// the queue's News (the watch's news), so an unchanged answer costs one comparison every reconcile and
// nothing on the bus. And on any node when how the apply went changed — refused, clean, or which resources it
// did not apply — since that is what the mesh raises a machine that does not apply from, and clears it by
// (novox/hq issue 233).
func reconcileKept(opts options, signer ed25519.PublicKey, sched *apply.Scheduler,
say link.Announce) link.Reconcile {
return func(ctx context.Context) (link.Report, bool) {
report, err := reapplyKept(ctx, opts, signer, sched, say)
if errors.Is(err, store.ErrNothingDeclared) {
// Nothing to hold this machine to yet. Ordinary on a node that has enrolled and not
// been assigned anything.
return link.Report{}, false
}
if err != nil {
say("cannot re-apply what this node was told: " + err.Error())
return link.Report{}, false
}
switch {
case report.Refused != "":
say("what this node was last told no longer applies: " + report.Refused)
case len(report.Failed) > 0:
say(fmt.Sprintf("holding this machine: %d applied, and %v", len(report.Applied), report.Failed))
}
return report, true
}
}
// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked.
//
// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline
// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a
// converged node — which holds nothing and found no firewall — could speak only in reply to a
// declaration, while the mesh composes no declaration for a node that has not said which links face
// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them.
//
// A refused report says nothing about the machine, so it is not news; the refusal is said on the
// console where a person reads it.
func worthSaying(report link.Report) bool {
if report.Refused != "" {
return false
}
return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0 ||
len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0 ||
len(report.Rollbacks) > 0 || len(report.Busy) > 0
}
// announceOr is what the apply writes its detail with, given what the caller has to say things with.
//
// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
@@ -2043,25 +1621,6 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
return applyAndKeepHeld(ctx, opts, raw, signed, sched, say)
}
// reapplyKept re-applies what this node was last told, **read once it is this apply's turn**.
//
// Read before waiting, it was whatever had been kept when the timer fired — and a declaration
// the link was applying at that moment is kept only once its apply ends. The reconcile then waited
// for that apply and applied the older declaration over it: a module assigned a second earlier was
// given back on the spot, and the report it sent named a declaration the mesh no longer recorded
// as sent, so a plan waiting on this machine waited for a report that could not come (novox/hq
// issues 257, 261). Read under the lock, it is always the latest the mesh said.
func reapplyKept(ctx context.Context, opts options, signer ed25519.PublicKey,
sched *apply.Scheduler, say link.Announce) (link.Report, error) {
applying.Lock()
defer applying.Unlock()
declared, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
if err != nil {
return link.Report{}, err
}
return applyAndKeepHeld(ctx, opts, declared, nil, sched, say), nil
}
// applyAndKeepHeld is applyAndKeep for a caller already holding applying.
func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *store.Declared,
sched *apply.Scheduler, say link.Announce) link.Report {
@@ -2075,36 +1634,10 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
if err != nil {
return link.Report{Refused: err.Error()}
}
order := link.Order{Epoch: declared.Epoch, Sequence: declared.Sequence}
// **Older than what this node applied is refused, before anything is read from the machine**
// (novox/hq to-be 45 §6, ADR 0227 rule 2): a controller that lost its lease and goes on sending.
// Read under the lock, so what it is compared with is what the last apply kept. Only a delivery:
// what a reconcile applies is what was kept.
if signed != nil {
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
if held, older := olderThanKept(kept, keptErr, order); older {
return link.Report{Declared: digestOf(raw), Order: order, OlderThan: &held,
Refused: fmt.Sprintf("this declaration was sent under lease epoch %d, sequence %d, and this "+
"node has applied epoch %d, sequence %d — from the controller holding the lease now. "+
"Refused whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)}
}
}
known, err := store.Load(opts.state)
if err != nil {
return link.Report{Refused: err.Error()}
}
// **Composed from an older view of the assignments than this machine applied is refused, before
// anything is read from the machine** (novox/hq issue 234). A newer sequence carrying an older view
// undeclared four assigned modules on 2026-10-04; the sequence cannot tell it, the generation can.
// Only a delivery: what a reconcile applies is what was kept.
if signed != nil {
if refused := olderGeneration(known, declared); refused != nil {
return link.Report{Declared: digestOf(raw), Order: order, OlderGeneration: refused,
Refused: olderGenerationWords(*refused)}
}
}
// A declaration from the mesh is the controller's word on the node's mode — the flip arrives
// as exactly that, the first converged declaration after adopted ones — and becomes the
// record. So does what this node kept: it is signed by the mesh and verified on load, where
@@ -2124,8 +1657,7 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
// 0486 §1, issue 1695): one object missing or differing refuses the declaration whole. The signed bytes are what is
// kept; only what is applied has the content in place of the reference.
if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
return link.Report{Declared: digestOf(raw), Order: order,
Refused: "a file of this declaration could not be had by its digest: " + err.Error()}
return link.Report{Refused: "a file of this declaration could not be had by its digest: " + err.Error()}
}
built, err := system.For(builtFor)
@@ -2169,8 +1701,6 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
updated.Mode = apply.ModeOf(declared)
// And the generation it came from, the same way, raised and never lowered (novox/hq issue 234).
updated.Generation = appliedGeneration(known, declared)
// Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's
// rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it
@@ -2204,7 +1734,44 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
sched.Sync(declared, held)
}
report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(),
report := factsOf(ctx, opts, declared, updated, outcome, held)
// Kept whichever way it went, so a node that is disconnected next minute still knows what it
// was told. Saved after applying rather than before: what is kept is what this node acted on.
var before []string
if signed != nil {
// The modules the declaration kept until now held, so a have key written for one the new declaration no longer
// holds is written removed even by a node-engine that started since (issue 1369).
if kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)); keptErr == nil {
if old, parseErr := declaration.ParseTrusted(kept.Declaration); parseErr == nil {
for _, f := range moduleFacts(old, apply.Report{}, nil) {
before = append(before, f.Module)
}
}
}
if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil {
report.Failed = map[string]string{"keeping the declaration": err.Error()}
}
}
if applyErr != nil {
report.Failed = map[string]string{"apply": applyErr.Error()}
}
report.Failed = withUnanswered(report.Failed, probe)
// And how each module of the declaration came out, for the have keys written beside this report (novox/hq ADR 0429
// §4, issue 1132): not on the wire.
report.Modules = moduleFacts(declared, outcome, applyErr)
report.Before = before
// Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire.
report.Unapplied = unapplied(declared, outcome, applyErr)
return report
}
// factsOf is what this node says of itself: its ports, its profile, how what runs here is, which links face outside,
// what it holds, its filters and, when adopted, its firewall and what is reachable. The facts are an event of the node
// to the controller (novox/hq ADR 0539 §9); declared is the declaration the node was last held to, updated its state,
// outcome what the last apply did (empty when nothing was applied), and held the ids it holds.
func factsOf(ctx context.Context, opts options, declared *declaration.Declaration, updated store.State,
outcome apply.Report, held map[string]bool) link.Report {
report := link.Report{Carried: carriedPorts(updated), Host: runningVersion(),
Rollbacks: standingRollbacks(opts.state), Witness: link.WitnessContract,
Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
@@ -2326,36 +1893,35 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
}
report.Applied = append(report.Applied, change.ID)
}
// Kept whichever way it went, so a node that is disconnected next minute still knows what it
// was told. Saved after applying rather than before: what is kept is what this node acted on.
var before []string
if signed != nil {
// The modules the declaration kept until now held, so a have key written for one the new declaration no longer
// holds is written removed even by a node-engine that started since (issue 1369).
if kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)); keptErr == nil {
if old, parseErr := declaration.ParseTrusted(kept.Declaration); parseErr == nil {
for _, f := range moduleFacts(old, apply.Report{}, nil) {
before = append(before, f.Module)
}
}
}
if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil {
report.Failed = map[string]string{"keeping the declaration": err.Error()}
}
}
if applyErr != nil {
report.Failed = map[string]string{"apply": applyErr.Error()}
}
report.Failed = withUnanswered(report.Failed, probe)
// And how each module of the declaration came out, for the have keys written beside this report (novox/hq ADR 0429
// §4, issue 1132): not on the wire.
report.Modules = moduleFacts(declared, outcome, applyErr)
report.Before = before
// Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire.
report.Unapplied = unapplied(declared, outcome, applyErr)
return report
}
// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs here
// is, and the haves of the modules the declaration it last applied holds. It applies nothing.
func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, say link.Announce) {
known, err := store.Load(opts.state)
if err != nil {
say("cannot say this node's facts: " + err.Error())
return
}
declared := &declaration.Declaration{}
if raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer); err == nil {
if d, err := declaration.Parse(raw); err == nil {
declared = d
}
}
held := map[string]bool{}
for _, h := range known.Held {
held[h.ID] = true
}
report := factsOf(ctx, opts, declared, known, apply.Report{}, held)
report.Modules = moduleFacts(declared, apply.Report{}, nil)
if report.Modules == nil {
report.Modules = []link.ModuleFact{}
}
session.SayFacts(ctx, report)
}
// waitForEnrolment returns this node's identity, waiting for one if it has none.
//
// It applies the carried bundle first, if there is one, because that is what a first node does
@@ -2488,92 +2054,6 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
os.Exit(0)
}
// refuseOlder refuses a declaration from the mesh that is older than the one this node holds.
//
// **By sequence, not by arrival** (novox/hq 04-ISSUES/107). What the host kept is the last thing
// the mesh said, signed; a declaration whose sequence is lower was composed before it, whatever
// order they arrived in — a backlog drained after the node was away, or a broker that split a burst.
// Applying it would make the machine into something the mesh had already moved past, which is the
// incident of issue 104 by another door.
//
// Only when both sides claim an order. A declaration with no sequence is one an older controller
// sent, and one kept with no sequence is one this host received before it understood them; in either
// case there is no order to compare, and refusing on a guess would strand the node the moment the
// controller is older than the host. Equal is the same declaration again, which reconciling is for.
//
// **And by the lease epoch first** (novox/hq to-be 45 §6): a declaration sent under an older epoch than
// the one kept here is a controller that lost its lease, whatever its sequence — the same rule the
// link's deliveries are held to (olderThanKept). Where both claim an epoch it alone decides, because a
// new lease holder's sequence says nothing against the last one's.
func refuseOlder(kept store.Declared, keptErr error, order link.Order) error {
if held, older := olderThanKept(kept, keptErr, order); older {
return fmt.Errorf("this declaration was sent under lease epoch %d, sequence %d, and the one kept "+
"here under epoch %d, sequence %d: a controller that no longer holds the lease sent it. Refused "+
"whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)
}
sequence := order.Sequence
if sequence == 0 || keptErr != nil {
return nil
}
last, err := declaration.ParseTrusted(kept.Declaration)
if err != nil || last.Sequence == 0 || (order.Epoch > 0 && last.Epoch > 0) {
return nil
}
if sequence < last.Sequence {
return fmt.Errorf("this declaration is older than what the mesh last said to this node: it "+
"is sequence %d, and the one kept here is %d. It arrived late — a backlog, or a broker "+
"that split a burst — and applying it would make this machine into something the mesh has "+
"already moved past. Refused whole; nothing was applied", sequence, last.Sequence)
}
return nil
}
// olderGeneration is the refusal of a declaration composed from an older assignment generation than
// this machine applied, nil when it is not one (novox/hq issue 234).
//
// Only when both claim a generation: a declaration without one is from a controller that claims none,
// and a machine that applied none has nothing to hold it to — refusing on a guess would strand every
// machine the moment the controller is older than the node-engine. Equal is not older: a push of the
// same assignments is the ordinary case. A put-back carries the generation of what it puts back and
// says so; a gate putting a machine back is the one sender that may say something older.
func olderGeneration(known store.State, d *declaration.Declaration) *link.GenerationRefusal {
if d.PutBack || d.Generation <= 0 || known.Generation <= 0 || d.Generation >= known.Generation {
return nil
}
return &link.GenerationRefusal{Generation: d.Generation, Applied: known.Generation}
}
// olderGenerationWords is a generation refusal as the report and a person read it.
func olderGenerationWords(r link.GenerationRefusal) string {
return fmt.Sprintf("this declaration was composed from assignment generation %d, and this node applied "+
"generation %d: it was composed from a view of the assignments the mesh has moved past, and applying it "+
"would undeclare what the mesh still assigns here (novox/hq issue 234). Refused whole; nothing was "+
"applied", r.Generation, r.Applied)
}
// appliedGeneration is the highest generation applied once this declaration is: raised by it, never
// lowered — a put-back of an older generation leaves it where it was (novox/hq issue 234).
func appliedGeneration(known store.State, d *declaration.Declaration) int64 {
return max(known.Generation, d.Generation)
}
// olderThanKept says whether a declaration of this order is older than the one this node kept — the
// last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept
// declaration that cannot be read claims no order: what it is compared with is unknown, and refusing
// on a guess would hold the machine off everything the mesh sends until a person looked. That kept
// file is read, and refused by name, by the next reconcile.
func olderThanKept(kept store.Declared, keptErr error, order link.Order) (link.Order, bool) {
if keptErr != nil || order.Epoch <= 0 {
return link.Order{}, false
}
last, err := declaration.ParseTrusted(kept.Declaration)
if err != nil {
return link.Order{}, false
}
held := link.Order{Epoch: last.Epoch, Sequence: last.Sequence}
return held, order.Older(held)
}
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
// report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161).
func profileAsReported(detected profile.Profile) map[string]any {
cmd/mesh-host/main_test.go
-271
View File
@@ -3,7 +3,6 @@ package main
import (
"bytes"
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"os"
@@ -14,8 +13,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/bundle"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
@@ -151,77 +148,6 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) {
}
}
// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds
// or its firewall changed — which is how a predecessor still writing is caught, without a report
// every five minutes saying nothing new.
func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
w := &adoptionWatch{}
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}}
if !w.changed(held) {
t.Fatal("the first report of a hold was not said")
}
again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}}
if w.changed(again) {
t.Error("the same holds in another order were said again")
}
rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}}
if !w.changed(rewritten) {
t.Error("a held file rewritten by something else was not said")
}
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
t.Error("a changed firewall was not said")
}
// What filters the machine is part of it (novox/hq ADR 0168): a predecessor's chain removed by
// hand, or the found firewall enabled again, is said without being asked.
filtered := link.Report{Firewall: "none", Held: rewritten.Held,
Filters: []link.Filter{{Where: "chain HAL-MESH-ONLY (iptables-legacy)", Owner: "other", Refuses: "-j DROP"}}}
if !w.changed(filtered) {
t.Error("a filter appearing was not said")
}
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
t.Error("a filter removed was not said")
}
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held, FoundFirewall: &link.FoundFirewall{Kind: "ufw", Active: true}}) {
t.Error("the found firewall coming back was not said")
}
if !worthSaying(link.Report{Filters: filtered.Filters}) {
t.Error("a report carrying only what filters the machine is not worth saying")
}
}
// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue
// tells the watch every report the broker took (link.Queue.Heard).
func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) {
w := &adoptionWatch{}
report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
heard := w.said
heard(report)
if w.changed(report) {
t.Error("a reconcile repeated what the mesh had just heard")
}
}
func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) {
// The controller previews a flip from what the node last said is reachable; a port that opened
// since must reach it without waiting for the next delivery (novox/hq ADR 0100).
w := &adoptionWatch{}
before := link.Report{Firewall: "ufw", Reachable: []link.Reach{
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
if !w.changed(before) {
t.Fatal("the first report was not said")
}
reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
if w.changed(reordered) {
t.Error("the same reachable set was said again")
}
opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable,
link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})}
if !w.changed(opened) {
t.Error("a newly published port was not said")
}
}
// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
// what the other recorded.
@@ -270,24 +196,6 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
}
}
// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued
// and lost — the link down when the reconcile spoke — it must be said again.
func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
w := &adoptionWatch{}
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}}
if !w.differs(held) {
t.Fatal("the first report of a change was not new")
}
// The link was down: nothing published it, so nothing says it was said.
if !w.differs(held) {
t.Error("a change that never reached the mesh was counted as said")
}
w.said(held)
if w.differs(held) {
t.Error("a change the mesh was told was said again")
}
}
// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
// the point of application, whichever command delivered it, naming both — an adopted control-node
// once applied its converged genesis bundle and closed itself for forty-five minutes.
@@ -468,125 +376,6 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
}
}
// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to
// that — never to the bundle the host carries, which genesis consumed.
func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) {
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
opts := stateWithMode(t, store.ModeAdopted)
controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said,
Signature: ed25519.Sign(controllerPrivate, said)}); err != nil {
t.Fatal(err)
}
// Told, and unable to prove by whom: refused, and the bundle is not applied in its place.
_, _, _, err = reconcileSource(opts)
if err == nil || !strings.Contains(err.Error(), "not applied in its place") {
t.Errorf("a node that cannot prove what it was told fell back to something: %v", err)
}
mine, err := identity.Generate("workstation")
if err != nil {
t.Fatal(err)
}
mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0",
Signer: controllerPublic, Password: "issued"}
if err := identity.Save(identity.Path(opts.state), mine); err != nil {
t.Fatal(err)
}
d, raw, from, err := reconcileSource(opts)
if err != nil {
t.Fatal(err)
}
if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil {
t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw))
}
// And before the mesh has said anything: the bundle, as it always was. A test binary carries
// only the placeholder, and asking for it is what proves the path.
if err := os.Remove(store.DeclaredPath(opts.state)); err != nil {
t.Fatal(err)
}
_, _, _, err = reconcileSource(opts)
if !errors.Is(err, bundle.ErrEmpty) {
t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
}
}
// **A machine says which links face outside without being asked.**
//
// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
// could speak only when a declaration arrived — and a declaration cannot be composed until it has
// spoken. A machine waiting for a push that is waiting for the machine.
func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
w := &adoptionWatch{}
first := link.Report{Firewall: "none"}
if !w.differs(first) {
t.Fatal("the first report should differ from nothing")
}
w.said(first)
// Only the links changed, and nothing about adoption.
learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
if !w.differs(learnt) {
t.Fatal("a machine that has just learnt which links face outside would never say so, " +
"and could then never be sent a filter")
}
w.said(learnt)
if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
}
// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
// filter is written around the old one until it is.
if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
}
}
// **A converged machine can say which links face outside, unasked.**
//
// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
// sat silent while the control plane refused to send them a filter.
func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
converged := link.Report{Outward: []string{"eth0"}}
if !worthSaying(converged) {
t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
"sent a filter — a machine waiting for a push that is waiting for the machine")
}
// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
if !worthSaying(link.Report{Firewall: "ufw"}) {
t.Fatal("an adopted machine no longer says which firewall it found")
}
if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
t.Fatal("an adopted machine no longer says what it holds")
}
// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
// about nothing.
if worthSaying(link.Report{}) {
t.Fatal("a reconcile with nothing to say speaks anyway")
}
// A refused report says nothing about the machine; the refusal is for the console.
if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
t.Fatal("a refused report is offered as news about the machine")
}
}
// **A host running as a service says what its apply did.**
//
// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
@@ -611,63 +400,3 @@ func TestTheApplysLogIsNeverNil(t *testing.T) {
t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
}
}
// Defends novox/hq issues 257 and 261: a reconcile due while the link applies a newer declaration
// applies that newer one once its turn comes — never the one kept when its timer fired. Read before
// waiting, it gave back a module assigned a second earlier, and reported a declaration the mesh no
// longer recorded as sent.
func TestAReconcileAppliesWhatWasKeptWhenItsTurnComes(t *testing.T) {
built, err := system.For("arch")
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
t.Skip("this machine is not one these tests can apply on")
}
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
target := filepath.Join(dir, "a.conf")
kept := func(content string) store.Declared {
body := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + target +
`","content":"` + content + `\n"}]}`)
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
}
if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("older")); err != nil {
t.Fatal(err)
}
// The link is applying: the reconcile's timer fires and it waits its turn.
applying.Lock()
done := make(chan error, 1)
go func() {
_, err := reapplyKept(context.Background(), opts, public, nil, nil)
done <- err
}()
time.Sleep(50 * time.Millisecond)
// The link's apply ends by keeping the newer declaration, and lets go.
if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("newer")); err != nil {
applying.Unlock()
t.Fatal(err)
}
applying.Unlock()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(10 * time.Second):
t.Fatal("the reconcile never ran once the node was free")
}
got, err := os.ReadFile(target)
if err != nil {
t.Fatal(err)
}
if string(got) != "newer\n" {
t.Errorf("the reconcile applied %q, what was kept before the newer declaration", got)
}
}
cmd/mesh-host/meshcli.go
+2 -2
View File
@@ -11,14 +11,14 @@ import (
// serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account
// from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served
// is said once, and this engine goes on applying; mesh-cli then says the engine does not answer.
func serveMeshCLI(ctx context.Context, queue *link.Queue, say func(string)) {
func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) {
if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil {
say("mesh-cli is not answered on this machine: " + err.Error())
}
}
// forwardOver asks the controller over the queue's link, and follows the line to what came of it.
func forwardOver(queue *link.Queue) meshcli.Forward {
func forwardOver(queue *link.Session) meshcli.Forward {
return func(ctx context.Context, asked meshcli.Asked) cli.Answer {
return meshcli.AskTheController(ctx, queue.AskController, asked)
}
cmd/mesh-host/movesecret.go
+1 -1
View File
@@ -27,7 +27,7 @@ type vaultAsk func(verb string, args any) (json.RawMessage, error)
// moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over
// the link open now.
func moveSecretFrom(statePath string, queue *link.Queue) link.MoveSecretFunc {
func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc {
return func(m link.MoveSecret) (link.MovedSecret, error) {
key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath))
if err != nil {
cmd/mesh-host/nodeclare_test.go
+42
View File
@@ -0,0 +1,42 @@
package main
import (
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
)
// R6-A AC8: nothing in the node-engine receives, orders or reconciles a whole declaration (novox/hq ADR 0539 §9, ADR
// 0429 §8): no package subscribes to a node's declare subject or names the deleted symbols.
func TestNoPackageHearsOrOrdersAWholeDeclaration(t *testing.T) {
deleted := []string{
"DeclareSubject", "mesh.node.\" + node + \".declare", "ReconcileDue", "ReconcileEvery", "olderThanKept",
"refuseOlder", "olderGeneration", "appliedGeneration", "reconcileKept", "reapplyKept", "adoptionWatch",
"unsaidFile", "numbersFile", "func gather(", "func handleBody(", "Supersedes(", "ReportSequence",
"RefusedOlder", "ReadsGeneration", "ReadsModules", "GenerationRefusal", "LeftOutModuleOf",
}
root := filepath.Join("..", "..")
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") {
return nil
}
body, err := os.ReadFile(path)
if err != nil {
return err
}
for _, word := range deleted {
if strings.Contains(string(body), word) {
t.Errorf("%s still names %q", path, word)
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
cmd/mesh-host/order_test.go
-95
View File
@@ -1,95 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq
// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said,
// "older" could not.
func keptWith(t *testing.T, sequence int64) store.Declared {
t.Helper()
return keptAt(t, link.Order{Sequence: sequence})
}
func keptAt(t *testing.T, order link.Order) store.Declared {
t.Helper()
body, err := json.Marshal(map[string]any{
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence,
"epoch": order.Epoch,
})
if err != nil {
t.Fatal(err)
}
return store.Declared{Declaration: body, Signature: []byte("x")}
}
func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5})
if err == nil {
t.Fatal("sequence 5 was accepted over a kept 7")
}
if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") {
t.Fatalf("the refusal does not say what it is: %v", err)
}
}
func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil {
t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
}
// Equal is the same declaration again, which reconciling is for.
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil {
t.Fatalf("the same sequence was refused: %v", err)
}
}
func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
// An older controller sends none; a host that received before it understood them kept none.
// Refusing on a guess would strand a node the moment the controller is older than the host.
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil {
t.Fatalf("a declaration claiming no order was refused: %v", err)
}
if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil {
t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
}
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil {
t.Fatalf("a first declaration was refused: %v", err)
}
}
// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost
// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence.
func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) {
kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3})
err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12})
if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") {
t.Fatalf("an older epoch was not refused by name: %v", err)
}
if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil {
t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err)
}
if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil {
t.Fatal("a lower sequence under the same epoch was accepted")
}
// A controller with no lease — older, or rolled back to a build from before it — is today's
// behaviour: no epoch compared.
if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil {
t.Fatalf("a declaration with no epoch was refused for one: %v", err)
}
if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older ||
held != (link.Order{Epoch: 57, Sequence: 3}) {
t.Fatalf("olderThanKept said %v, holding %+v", older, held)
}
// What is kept cannot be read: no order to compare, so nothing is refused on a guess.
if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil,
link.Order{Epoch: 41, Sequence: 1}); older {
t.Fatal("refused against a kept declaration nobody can read")
}
}
cmd/mesh-host/probe_test.go
-81
View File
@@ -9,7 +9,6 @@ import (
"testing"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/system"
)
@@ -127,83 +126,3 @@ func TestAPackageManagerThatIsNotThereRefusesWhole(t *testing.T) {
t.Error("the refused declaration touched the machine")
}
}
// Defends the review of novox/hq issue 233: **a re-apply whose result changed is said unasked**. A machine
// reconciles every five minutes and speaks only when its account is news; how the apply went was not part of
// it, so a package manager that stopped answering between deliveries was never said, and one that answered
// again never cleared. Now a failure is said once, the same failure again is not, and a clean re-apply after it
// is — which is what clears the mesh's condition.
func TestAReapplyWhoseResultChangedIsSaidUnasked(t *testing.T) {
dir := t.TempDir()
conf := filepath.Join(dir, "elsewhere", "pacman.conf") // not declared: the apply cannot make it answer
pacmanAnsweringOnce(t, conf)
opts := options{state: filepath.Join(dir, "state.json")}
raw := []byte(`{"declaration":1,"resources":[` +
`{"id":"motd","type":"file","path":"` + filepath.Join(dir, "motd") + `","content":"hello\n"},` +
`{"id":"pacman.base","type":"package","package":"base"}]}`)
w := &adoptionWatch{}
// Clean once, as it was before: base installed while pacman answered.
if err := os.MkdirAll(filepath.Dir(conf), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil {
t.Fatal(err)
}
clean := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
if faulty(clean) {
t.Fatalf("the first apply was not clean: %+v", clean)
}
w.said(clean)
if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) {
t.Error("a clean re-apply after a clean one was said, so every machine speaks every five minutes")
}
// The configuration goes: the re-apply fails, and that is news once.
if err := os.Remove(conf); err != nil {
t.Fatal(err)
}
failed := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
if !faulty(failed) {
t.Fatalf("the re-apply did not fail: %+v", failed)
}
if !w.news(failed) {
t.Fatal("a re-apply that failed after a clean one was not said, so the mesh never hears the machine fails")
}
w.said(failed)
if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) {
t.Error("the same failure was said again on the next re-apply")
}
// It comes back: the clean re-apply is news, and is what clears the mesh's condition.
if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil {
t.Fatal(err)
}
again := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
if faulty(again) {
t.Fatalf("the re-apply did not come back clean: %+v", again)
}
if !w.news(again) {
t.Fatal("a clean re-apply after a failed one was not said, so the mesh's condition never clears")
}
}
// A refusal is news too, once, and a reconcile with nothing to say after a clean one stays silent.
func TestARefusalIsSaidOnceAndNothingIsNotSaid(t *testing.T) {
w := &adoptionWatch{}
w.said(link.Report{})
if w.news(link.Report{}) {
t.Error("nothing was said again")
}
refused := link.Report{Refused: "pacman does not answer here"}
if !w.news(refused) {
t.Fatal("a refusal was not said")
}
w.said(refused)
if w.news(link.Report{Refused: "pacman does not answer here, at another time"}) {
t.Error("the same refusal in other words was said again")
}
if !w.news(link.Report{}) {
t.Error("a clean apply after a refusal was not said")
}
}
cmd/mesh-host/unsaid_test.go
-64
View File
@@ -1,64 +0,0 @@
package main
import (
"os"
"path/filepath"
"reflect"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// The last apply's report, kept beside the state until the mesh takes it (novox/hq issue 264).
// What a host that died between applying and reporting kept is read back by the next one exactly —
// the digest and the outcome, never a new account of the machine.
func TestTheKeptReportSurvivesTheHost(t *testing.T) {
state := filepath.Join(t.TempDir(), "state.json")
made := link.Report{Declared: "sha256:d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"},
Host: "v2"}
if err := unsaidBeside(state).Keep(made); err != nil {
t.Fatal(err)
}
kept, ok, err := unsaidBeside(state).Pending()
if err != nil || !ok {
t.Fatalf("nothing kept: %v", err)
}
if !reflect.DeepEqual(kept, made) {
t.Fatalf("kept %+v, made %+v", kept, made)
}
// Taken about another declaration, it stays; taken about this one, it goes.
if err := unsaidBeside(state).Said("sha256:other"); err != nil {
t.Fatal(err)
}
if _, ok, _ := unsaidBeside(state).Pending(); !ok {
t.Fatal("a report about another declaration cleared this one")
}
if err := unsaidBeside(state).Said("sha256:d1"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(store.UnsaidPath(state)); !os.IsNotExist(err) {
t.Fatalf("the report was taken and is still kept: %v", err)
}
}
// A node that never applied anything has nothing to say again; and an apply refused before its
// declaration was read clears an older report, which would otherwise be re-said after it.
func TestNothingKeptWhenNothingWasApplied(t *testing.T) {
state := filepath.Join(t.TempDir(), "state.json")
if _, ok, err := unsaidBeside(state).Pending(); ok || err != nil {
t.Fatalf("a node that applied nothing has a report to say: %v %v", ok, err)
}
if err := unsaidBeside(state).Keep(link.Report{Declared: "sha256:d1"}); err != nil {
t.Fatal(err)
}
if err := unsaidBeside(state).Keep(link.Report{Refused: "forged"}); err != nil {
t.Fatal(err)
}
if _, ok, _ := unsaidBeside(state).Pending(); ok {
t.Fatal("an older report is kept after a later declaration was refused")
}
}
cmd/mesh-host/vaultbox_nats_test.go
+5 -6
View File
@@ -22,7 +22,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
@@ -61,14 +60,14 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(
}
_ = tap.Flush()
// The engine hears its declaration on its own subject.
const node = "vaultbox"
// The engine hears the composition on a subject of the test (the node agent reads it from its bucket).
const subject = "mesh.test.vaultbox.composition"
engine, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer engine.Close()
heard, err := engine.SubscribeSync(link.DeclareSubject(node))
heard, err := engine.SubscribeSync(subject)
if err != nil {
t.Fatal(err)
}
@@ -92,7 +91,7 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(
t.Fatal(err)
}
defer controller.Close()
if err := controller.Publish(link.DeclareSubject(node), body); err != nil {
if err := controller.Publish(subject, body); err != nil {
t.Fatal(err)
}
_ = controller.Flush()
@@ -123,7 +122,7 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(
mu.Lock()
all := strings.Join(crossed, "\n")
mu.Unlock()
if !strings.Contains(all, link.DeclareSubject(node)) || !strings.Contains(all, "vault-sealed") {
if !strings.Contains(all, subject) || !strings.Contains(all, "vault-sealed") {
t.Fatalf("the listener heard nothing of the declaration:\n%s", all)
}
for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} {
cmd/mesh-host/witnessing.go
+10 -11
View File
@@ -41,8 +41,9 @@ func rolledBackHosts(statePath string) []string {
return upgrade.RolledBackVersions(verdicts)
}
// proving is this host waiting for the mesh to take a report it made under its own build — the
// evidence its launcher's trial waits for — and acting on it once.
// proving is this host waiting to write a have key under its own build — the evidence its launcher's trial waits for
// (novox/hq ADR 0539 §9, ADR 0227 §8: the component replaced is never the only witness of its successor) — and acting
// on it once.
type proving struct {
statePath string
version string
@@ -56,16 +57,14 @@ func unproved(statePath, version string) bool {
return err != nil || known != version
}
// heard is told every account of the machine the mesh has taken. The first one about a declaration,
// made by this build, proves it.
func (p *proving) heard(r link.Report) {
if r.Declared == "" || r.Refused != "" || r.Host != p.version {
return
}
// haveWritten is told when this build has written a have key, or found the bucket already holding what it would write.
// The first proves it. **A declaration's report does not**: a node-engine that receives none is not on trial for it, and
// one that cannot write its haves is what the launcher keeps rolling back.
func (p *proving) haveWritten() {
p.once.Do(func() {
retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version)
if err != nil {
p.say(fmt.Sprintf("this host reported under its own build %s, and proving it was not complete: %v", p.version, err))
p.say(fmt.Sprintf("this host wrote its first have under its own build %s, and proving it was not complete: %v", p.version, err))
}
if len(retired) > 0 {
p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", ")))
@@ -146,7 +145,7 @@ func launcherReplaced(pid int, proc, path string) bool {
}
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue, say func(string)) {
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
host, _ := os.Hostname()
w := &witness.Watcher{
Root: apply.DaemonRoot(), Node: node, Host: host,
@@ -159,7 +158,7 @@ func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue
},
// Said now, not at the next report: the verdict is on every report from here, and the mesh
// is asked for one at once.
Concluded: func(link.Rollback) { queue.ReportAsked() },
Concluded: func(link.Rollback) { queue.FactsAsked(ctx) },
Say: say,
}
w.Watch(ctx)
cmd/mesh-host/witnessing_test.go
+15 -23
View File
@@ -13,9 +13,9 @@ import (
"github.com/novox/mesh-host/internal/upgrade"
)
// The first account the mesh takes from this build, about a declaration, proves it to the launcher;
// nothing else does — a refusal, a report about no declaration, another build's report (to-be 45 §8).
func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) {
// The first have this build writes proves it to the launcher; nothing else does — not a report about a declaration, not
// a facts event, not the haves of another build (novox/hq ADR 0539 §9, ADR 0227 §8). R6-A AC7.
func TestOnlyAHaveUnderThisBuildProvesIt(t *testing.T) {
state := filepath.Join(t.TempDir(), "state.json")
t.Setenv("MESH_HOST_LIBEXEC", t.TempDir())
// Never the machine's own engine placed by hand: proving renews it (issue 356).
@@ -27,22 +27,20 @@ func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) {
var said []string
p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }}
for _, r := range []link.Report{
{Host: "2.0", Refused: "no"},
{Host: "2.0"},
{Host: "1.0", Declared: "abc"},
} {
p.heard(r)
if !unproved(state, "2.0") {
t.Fatalf("%+v proved this build", r)
}
if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" {
t.Fatalf("%+v renewed the engine placed by hand", r)
}
// Facts said, a link opened, a declaration of the old kind reported: none of it is a have.
if !unproved(state, "2.0") {
t.Fatal("a build that wrote no have is proved")
}
p.heard(link.Report{Host: "2.0", Declared: "abc"})
if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" {
t.Fatal("the engine placed by hand was renewed before any have")
}
// The session tells the proof when it has a have key saying what it would write.
session := &link.Session{HaveWritten: p.haveWritten}
_ = session
p.haveWritten()
if unproved(state, "2.0") {
t.Fatalf("an account of a declaration under this build did not prove it (%v)", said)
t.Fatalf("the first have under this build did not prove it (%v)", said)
}
if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" {
t.Fatalf("known-good is %q", got)
@@ -78,12 +76,6 @@ func TestAStandingRollbackIsSaidOnEveryReport(t *testing.T) {
got[0].Outcome != link.RolledBack {
t.Fatalf("what the report says is %+v", got)
}
if !worthSaying(link.Report{Rollbacks: got}) {
t.Fatal("a reconcile with a rollback standing does not say it")
}
if adoptionFingerprint(link.Report{Rollbacks: got}) == adoptionFingerprint(link.Report{}) {
t.Fatal("a rollback reached or ended is not news to the reconcile")
}
}
// The launcher running this host is known to have been replaced only when it certainly was: a shell
internal/apply/apply.go
-18
View File
@@ -21,7 +21,6 @@ import (
"os"
"os/exec"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
@@ -561,18 +560,6 @@ func applyScoped(
protecting = append(protecting, orphan)
continue
}
// **A module the mesh left out is not a module the mesh removed** (novox/hq ADR 0163, rule
// 6): its resources are absent because a setting stored for it cannot compose, and the
// mesh said so by name. What the host wrote for it stays as it is, recorded, until the
// module is declared again or unassigned.
if module, left := d.LeftOutModuleOf(orphan.ID); left {
report.Outcomes = append(report.Outcomes, Outcome{
ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
Action: "unchanged", Detail: "kept: " + module + " was left out of this declaration by the mesh, not removed",
})
log(fmt.Sprintf(" kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
continue
}
if by, replaced := replacedBy[orphan.ID]; replaced {
handover[by] = append(handover[by], orphan)
continue
@@ -616,11 +603,6 @@ func applyScoped(
if declared[h.ID] || (scope != nil && h.Module != scope.Module) {
continue
}
if slices.Contains(d.LeftOut, h.Module) {
// Left out, not unassigned (ADR 0163, rule 6): still held for the module, as the
// mesh asked.
continue
}
known.Release(h.ID)
report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target,
Action: "forgotten", Detail: "no longer declared; left as found"})
internal/apply/left_out_test.go → internal/apply/containers_kept_test.go
-63
View File
@@ -75,69 +75,6 @@ func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) {
var specOfLast string
// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163,
// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held.
// A module simply absent is removed as it always was.
func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) {
var removed []string
gone := map[string]bool{}
run := func(_ context.Context, name string, args ...string) (string, error) {
args = oldShape(name, args)
if name != "docker" {
return "", nil
}
switch args[0] {
case "info":
return "29.0.0\n", nil
case "rm":
removed = append(removed, args[len(args)-1])
gone[args[len(args)-1]] = true
case "container":
if gone[args[len(args)-1]] {
return "", errors.New("no such container")
}
return "true\tspec", nil
}
return "", nil
}
known := store.State{
Resources: []store.Applied{
{ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared},
{ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared},
},
Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}},
}
d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[
{"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"}
]}`)
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(removed) != 1 || removed[0] != "old" {
t.Fatalf("removed %v; only the module that is absent goes", removed)
}
if _, kept := state.At("container", "web"); !kept {
t.Fatal("the left-out module's record was forgotten")
}
if _, held := state.HeldAt("web.page"); !held {
t.Fatal("the left-out module's hold was released")
}
said := false
for _, o := range report.Outcomes {
if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") {
said = true
}
if o.ID == "web.server" && o.Action != "unchanged" {
t.Errorf("the left-out module's container was %s", o.Action)
}
}
if !said {
t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes)
}
}
// A container's capabilities reach the runtime and are part of its spec (novox/hq ADR 0170).
func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
var ran []string
run := func(_ context.Context, name string, args ...string) (string, error) {
internal/link/asked_test.go
-130
View File
@@ -1,130 +0,0 @@
package link
import (
"context"
"strings"
"testing"
"time"
)
// The `report` verb (novox/hq to-be 45 §6), what healer H1 asks a machine whose send went unreported:
// the node says again what it last applied, as an ordinary report through the one queue.
// **Asked, the reconcile's account is said even when it is not news** — the mesh asked because it does
// not know, and silence would be the very fault it is asking about.
func TestAskedToReportSaysWhatWasLastAppliedEvenWhenNotNews(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
var lines []string
q := aQueue(m, (&applies{}).apply, &keptInMemory{})
q.Say = func(s string) { lines = append(lines, s) }
q.Reconcile = func(context.Context) (Report, bool) {
return Report{Declared: "d-kept", Order: Order{Epoch: 7, Sequence: 12}, Applied: []string{"a"}}, true
}
q.News = func(Report) bool { return false }
q.attach(context.Background(), l)
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
q.ReconcileDue() // an ordinary reconcile: not news, not said
time.Sleep(50 * time.Millisecond)
if n := len(l.said()); n != 0 {
t.Fatalf("an ordinary reconcile that is not news said %d report(s)", n)
}
q.ReportAsked()
eventually(t, "asked, the node said nothing", func() bool { return len(l.said()) == 1 })
stop()
<-worker
r := l.said()[0]
if r.Declared != "d-kept" || r.Sequence != 12 || r.Epoch != 7 || r.ReportSequence == 0 || r.Node != m.Node {
t.Fatalf("the account said is not the one kept, in order: %+v", r)
}
if !strings.Contains(strings.Join(lines, "\n"), "asked by the mesh") {
t.Errorf("being asked is not said in the journal: %q", lines)
}
}
// **A delivery waiting answers the question better**: it is applied, once, and its report is the answer.
func TestAskedWhileADeliveryWaitsIsThatDeliverysReport(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
q := aQueue(m, a.apply, &keptInMemory{})
reconciled := 0
q.Reconcile = func(context.Context) (Report, bool) { reconciled++; return Report{}, true }
q.attach(context.Background(), l)
q.Deliver(ordered(t, key, "new", 7, 13))
q.ReportAsked()
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 })
time.Sleep(50 * time.Millisecond)
stop()
<-worker
if reconciled != 0 || len(l.said()) != 1 || l.said()[0].Sequence != 13 {
t.Fatalf("reconciled %d, said %+v", reconciled, l.said())
}
}
// askingLink is a quiet link the mesh can ask.
type askingLink struct {
*quietLink
asks chan struct{}
}
func (l askingLink) AskedToReport() <-chan struct{} { return l.asks }
// **The link hands the question to the queue**: asked on the link, the account is said on it.
func TestALinkThatIsAskedHandsItToTheQueue(t *testing.T) {
m, _ := aMember(t)
l := askingLink{quietLink: newQuietLink(), asks: make(chan struct{}, 1)}
q := aQueue(m, (&applies{}).apply, &keptInMemory{})
q.Reconcile = func(context.Context) (Report, bool) { return Report{Declared: "d-kept"}, true }
ctx, stop := context.WithCancel(context.Background())
defer stop()
worker := runWorker(ctx, q)
served := make(chan error, 1)
go func() { served <- serve(ctx, l, m, q, nil, time.Second) }()
l.asks <- struct{}{}
eventually(t, "asked on the link, nothing was said", func() bool {
for _, r := range l.said() {
if r.Declared == "d-kept" {
return true
}
}
return false
})
stop()
<-served
<-worker
}
// **Against a real server**: the question on the node's own subject reaches it, and another's does not.
func TestNatsTheMeshAsksANodeToReport(t *testing.T) {
conn, js := aBus(t)
l := &natsLink{conn: conn, js: js, node: "asked"}
l.hearAsks()
defer l.Close()
if err := conn.Publish(AskReportSubject("another"), []byte(`{}`)); err != nil {
t.Fatal(err)
}
if err := conn.Publish(AskReportSubject("asked"), []byte(`{"by":"the controller's healer H1"}`)); err != nil {
t.Fatal(err)
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
select {
case <-l.AskedToReport():
case <-time.After(5 * time.Second):
t.Fatal("the question on the node's own subject never reached it")
}
select {
case <-l.AskedToReport():
t.Fatal("asked twice: another node's question reached this one")
case <-time.After(200 * time.Millisecond):
}
if AskReportSubject("asked") != "mesh.node.asked.ask.report" {
t.Fatalf("the subject %q is not the one the controller asks on and grants", AskReportSubject("asked"))
}
}
internal/link/asktool_nats_test.go
+1 -1
View File
@@ -25,7 +25,7 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) {
t.Fatal(err)
}
q := &Queue{Membership: Membership{Node: node}}
q := &Session{Membership: Membership{Node: node}}
q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node})
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
defer cancel()
internal/link/handover_test.go
+1 -1
View File
@@ -184,7 +184,7 @@ func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
m, _ := aMember(t)
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)}
q := aQueue(m, (&applies{}).apply, &keptInMemory{})
q := &Session{Membership: m, Timeout: time.Second}
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
ctx, stop := context.WithCancel(context.Background())
served := make(chan error, 1)
internal/link/have.go
+17 -6
View File
@@ -97,7 +97,7 @@ func nodeFacts(r Report) delivery.HaveNode {
}
// noteApply takes what an apply's report says of the modules and the node.
func (q *Queue) noteApply(r Report) bool {
func (q *Session) noteApply(r Report) bool {
if r.Modules == nil {
return false
}
@@ -146,7 +146,7 @@ func (q *Queue) noteApply(r Report) bool {
}
// noteHealth takes a look's health, and says whether any module's changed.
func (q *Queue) noteHealth(h Health) bool {
func (q *Session) noteHealth(h Health) bool {
s := &q.haves
s.mu.Lock()
defer s.mu.Unlock()
@@ -170,7 +170,7 @@ func (q *Queue) noteHealth(h Health) bool {
}
// sayHave is a line in the node-engine's log, if it has one.
func (q *Queue) sayHave(s string) {
func (q *Session) sayHave(s string) {
if q.Say != nil {
q.Say(s)
}
@@ -180,7 +180,7 @@ func (q *Queue) sayHave(s string) {
// §4, issue 1132): each module's have, and the node's own facts. It reads the key first and writes nothing when the value
// is the same, otherwise writes over the revision it read, and creates the key when there is none. A write the bucket
// refuses is said and left: the next apply or look writes again, and the report was sent before this ran.
func (q *Queue) writeHaves(ctx context.Context, bus Bus) {
func (q *Session) writeHaves(ctx context.Context, bus Bus) {
hb, ok := bus.(HaveBus)
if !ok || bus == nil {
return
@@ -261,7 +261,7 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) {
// writeRemoved writes, once, each module the applied declaration no longer holds and this node-engine wrote a have key
// for, with the phase removed over the revision it reads (issue 1369). The key is never deleted, and a key that is not
// in the bucket is not created. False when the pass should stop. The caller holds the lock of the haves.
func (q *Queue) writeRemoved(ctx context.Context, b HaveBucket, node string) bool {
func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) bool {
s := &q.haves
gone := make([]string, 0, len(s.removed))
for module := range s.removed {
@@ -311,7 +311,7 @@ func (q *Queue) writeRemoved(ctx context.Context, b HaveBucket, node string) boo
// putHave reads a key and writes it unless same says it already holds the value. False when the pass should stop: the
// bucket failed, the error is said once, and the rest wait for the next pass.
func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool {
func (q *Session) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool {
if ctx.Err() != nil {
q.sayHave(fmt.Sprintf("the time for writing the have bucket ran out before %s; the next apply or look writes it", key))
return false
@@ -324,11 +324,22 @@ func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []b
q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err))
return false
case same(current):
q.haveWritten()
return true
}
if err := b.Set(ctx, key, value, revision); err != nil {
q.sayHave(fmt.Sprintf("could not write %s in the have bucket; the next apply or look writes it again: %v", key, err))
return false
}
q.haveWritten()
return true
}
// haveWritten tells the caller this node-engine has a have key in the bucket saying what it says now: written by it, or
// already holding the same. It is what proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9): the first
// have under its own build, not a declaration's report.
func (q *Session) haveWritten() {
if q.HaveWritten != nil {
q.HaveWritten()
}
}
internal/link/have_nats_integration_test.go
+14 -21
View File
@@ -4,7 +4,6 @@ package link
import (
"context"
"encoding/json"
"sync/atomic"
"testing"
"time"
@@ -32,26 +31,20 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) {
_ = kv.Purge(node + "." + name)
}
m, key := aMember(t)
m, _ := aMember(t)
m.Node = node
var failB atomic.Bool
var applied atomic.Int32
failB.Store(true)
apply := func(_ context.Context, raw, _ []byte) Report {
applied.Add(1)
var d struct{ Order }
_ = json.Unmarshal(raw, &d)
return Report{Declared: digest(raw), Order: d.Order, Applied: []string{"a.x", "b.x"},
facts := func() Report {
return Report{Applied: []string{"a.x", "b.x"},
Modules: []ModuleFact{{Module: "a", Build: "b-1"},
{Module: "b", Build: "b-2", Failed: failB.Load(), Detail: "b.x failed: no space"}},
Health: healthOf(map[string]string{"a": StateHealthy, "b": StateHealthy}),
Windows: []Window{{Step: "backup", Holds: []string{"db"}}}}
}
q := aQueue(m, apply, &keptInMemory{})
q := &Session{Membership: m, Timeout: 2 * time.Second}
q.attach(context.Background(), OverNATS{Conn: conn, JS: js})
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
defer func() { stop(); <-worker }()
ctx := context.Background()
revisions := func() map[string]uint64 {
out := map[string]uint64{}
@@ -62,14 +55,14 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) {
}
return out
}
arrives := func(what string, id string, seq int64) {
want := applied.Load() + 1
q.Deliver(ordered(t, key, id, 7, seq))
eventually(t, what, func() bool { return applied.Load() >= want })
arrives := func(what string) {
if !q.SayFacts(ctx, facts()) {
t.Fatal(what + ": the facts were not taken")
}
time.Sleep(300 * time.Millisecond)
}
arrives("the first apply", "one", 1)
eventually(t, "the three keys were never written", func() bool { return len(revisions()) == 3 })
arrives("the first facts")
eventuallyHeld(t, "the three keys were never written", func() bool { return len(revisions()) == 3 })
first := revisions()
a, _ := kv.Get(node + ".a")
@@ -87,7 +80,7 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) {
t.Fatalf("the node key holds %+v (%v)", hn, err)
}
arrives("the second apply", "two", 2)
arrives("the second facts")
time.Sleep(200 * time.Millisecond)
for name, rev := range revisions() {
if rev != first[name] {
@@ -96,8 +89,8 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) {
}
failB.Store(false)
arrives("the third apply", "three", 3)
eventually(t, "b was never updated", func() bool { return revisions()["b"] > first["b"] })
arrives("the third facts")
eventuallyHeld(t, "b was never updated", func() bool { return revisions()["b"] > first["b"] })
if now := revisions(); now["a"] != first["a"] || now["node"] != first["node"] {
t.Errorf("a key that did not change moved: %v from %v", now, first)
}
internal/link/have_test.go
+65 -52
View File
@@ -126,15 +126,15 @@ func healthOf(states map[string]string) *Health {
}
func appliedModules() Report {
return Report{Declared: "d1", Applied: []string{"a.x", "b.x"},
return Report{Applied: []string{"a.x", "b.x"},
Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}},
Health: healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})}
}
func aHaveQueue(l *haveLink) (*Queue, *[]string) {
func aHaveQueue(l *haveLink) (*Session, *[]string) {
var said []string
var mu sync.Mutex
q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) {
q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) {
mu.Lock()
said = append(said, s)
mu.Unlock()
@@ -148,7 +148,7 @@ func aHaveQueue(l *haveLink) (*Queue, *[]string) {
func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
a, b := l.have(t, "n1.a"), l.have(t, "n1.b")
if a.Phase != delivery.PhaseRunning || a.Build != "b-1" || a.Health != delivery.HealthOK {
t.Errorf("a: %+v", a)
@@ -168,7 +168,7 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) {
l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-0", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
q, _ := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
if l.kv["n1.a"].rev != 1 {
t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev)
}
@@ -176,7 +176,7 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) {
t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev)
}
before := l.setCount()
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
if l.setCount() != before {
t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before)
}
@@ -186,9 +186,9 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) {
func TestAModulesHealthFollowsTheLook(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
rep := Report{Declared: "d1", Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
rep := Report{Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
{Module: "c", Build: "b"}, {Module: "d", Build: "b"}}}
q.tell(context.Background(), rep, applied)
q.SayFacts(context.Background(), rep)
for _, m := range []string{"a", "b", "c", "d"} {
if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown {
t.Errorf("%s with no look is %s, want unknown", m, h)
@@ -230,7 +230,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
rep.Strays = []Stray{{Kind: "container", Name: "old"}}
rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
rep.Profile = map[string]any{"cores": 8}
q.tell(context.Background(), rep, applied)
q.SayFacts(context.Background(), rep)
if len(l.events) < 2 || l.events[0] != "report" {
t.Fatalf("the report was not first: %v", l.events)
}
@@ -255,7 +255,7 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
l := newHaveLink()
l.failing = true
q, said := aHaveQueue(l)
if !q.tell(context.Background(), appliedModules(), applied) {
if !q.SayFacts(context.Background(), appliedModules()) {
t.Fatal("the report did not go out")
}
if len(l.said()) != 1 {
@@ -267,7 +267,7 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
l.hmu.Lock()
l.failing = false
l.hmu.Unlock()
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
if l.have(t, "n1.a").Phase != delivery.PhaseRunning {
t.Error("the next apply did not write again")
}
@@ -276,39 +276,19 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
// A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong.
func TestNoBucketAndNoFactsWriteNothing(t *testing.T) {
plain := newQuietLink()
q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second}
q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second}
q.attach(context.Background(), plain)
if !q.tell(context.Background(), appliedModules(), applied) {
if !q.SayFacts(context.Background(), appliedModules()) {
t.Fatal("the report did not go out")
}
l := newHaveLink()
q2, _ := aHaveQueue(l)
q2.tell(context.Background(), Report{Refused: "no"}, applied)
q2.SayFacts(context.Background(), Report{Refused: "no"})
if l.setCount() != 0 {
t.Errorf("a refusal wrote %d keys", l.setCount())
}
}
// 1132/AC5: a reconcile whose report is not news still writes the have keys that differ, so a node-engine that
// restarted writes them again; and writes nothing when they are the same.
func TestASilentReconcileStillWritesTheHaves(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
q.Reconcile = func(context.Context) (Report, bool) { return appliedModules(), true }
q.reconcile(context.Background(), false)
if len(l.said()) != 0 {
t.Fatalf("a reconcile that is not news said %d reports", len(l.said()))
}
if l.have(t, "n1.a").Phase != delivery.PhaseRunning {
t.Error("the reconcile wrote no have")
}
before := l.setCount()
q.reconcile(context.Background(), false)
if l.setCount() != before {
t.Errorf("an unchanged reconcile wrote %d keys", l.setCount()-before)
}
}
// 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that
// fails ends the pass, and is said once.
func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) {
@@ -321,7 +301,7 @@ func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) {
q, said := aHaveQueue(l)
q.Timeout = 100 * time.Millisecond
start := time.Now()
if !q.tell(context.Background(), appliedModules(), applied) {
if !q.SayFacts(context.Background(), appliedModules()) {
t.Fatalf("%s: the report did not go out", name)
}
if took := time.Since(start); took > time.Second {
@@ -334,14 +314,14 @@ func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) {
l := newHaveLink()
l.failing = true
q, said := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
if l.binds != 1 || l.setTries != 1 || len(*said) != 1 {
t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said)
}
}
func onlyA() Report {
return Report{Declared: "d2", Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}},
return Report{Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}},
Health: healthOf(map[string]string{"a": StateHealthy})}
}
@@ -350,9 +330,9 @@ func onlyA() Report {
func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev
q.tell(context.Background(), onlyA(), applied)
q.SayFacts(context.Background(), onlyA())
b := l.have(t, "n1.b")
if b.Phase != delivery.PhaseRemoved || b.Build != "b-2" {
t.Errorf("b: %+v", b)
@@ -364,7 +344,7 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev)
}
before := l.setCount()
q.tell(context.Background(), onlyA(), applied)
q.SayFacts(context.Background(), onlyA())
if l.setCount() != before {
t.Errorf("a second apply without b wrote %d keys, removed is written once", l.setCount()-before)
}
@@ -374,13 +354,13 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
func TestARemovedModuleAddedBackIsWrittenWithItsNewPhase(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.tell(context.Background(), onlyA(), applied)
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
q.SayFacts(context.Background(), onlyA())
q.SayFacts(context.Background(), appliedModules())
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseFailed || b.Build != "b-2" {
t.Errorf("b after it came back: %+v", b)
}
q.tell(context.Background(), onlyA(), applied)
q.SayFacts(context.Background(), onlyA())
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
t.Errorf("b removed again: %+v", b)
}
@@ -392,19 +372,19 @@ func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) {
l := newHaveLink()
l.put(t, "n1.c", delivery.Have{V: delivery.Version, Build: "b-9", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
q, _ := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.tell(context.Background(), onlyA(), applied)
q.SayFacts(context.Background(), appliedModules())
q.SayFacts(context.Background(), onlyA())
if c := l.have(t, "n1.c"); c.Phase != delivery.PhaseRunning || l.kv["n1.c"].rev != 1 {
t.Errorf("c, which this node-engine never wrote, changed: %+v at %d", c, l.kv["n1.c"].rev)
}
// a key that vanished from the bucket is not created by the removal
l2 := newHaveLink()
q2, _ := aHaveQueue(l2)
q2.tell(context.Background(), appliedModules(), applied)
q2.SayFacts(context.Background(), appliedModules())
l2.hmu.Lock()
delete(l2.kv, "n1.b")
l2.hmu.Unlock()
q2.tell(context.Background(), onlyA(), applied)
q2.SayFacts(context.Background(), onlyA())
if _, there := l2.kv["n1.b"]; there {
t.Error("the removal created a key that was not there")
}
@@ -414,15 +394,15 @@ func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) {
func TestARefusedRemovalIsWrittenByTheNextPass(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
q.tell(context.Background(), appliedModules(), applied)
q.SayFacts(context.Background(), appliedModules())
l.hmu.Lock()
l.failing = true
l.hmu.Unlock()
q.tell(context.Background(), onlyA(), applied)
q.SayFacts(context.Background(), onlyA())
l.hmu.Lock()
l.failing = false
l.hmu.Unlock()
q.tell(context.Background(), onlyA(), applied)
q.SayFacts(context.Background(), onlyA())
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
t.Errorf("b: %+v", b)
}
@@ -437,7 +417,7 @@ func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T)
q, _ := aHaveQueue(l) // fresh: nothing written by this process
r := onlyA()
r.Before = []string{"a", "b", "c"}
q.tell(context.Background(), r, applied)
q.SayFacts(context.Background(), r)
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
t.Errorf("b: %+v", b)
}
@@ -448,3 +428,36 @@ func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T)
t.Errorf("a was written, revision %d", l.kv["n1.a"].rev)
}
}
// R6-A AC7: the first have this node-engine writes — or finds already saying what it would write — is told to the
// proof, which proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9); a write the bucket refuses is not.
func TestTheFirstHaveWrittenIsToldToTheProof(t *testing.T) {
l := newHaveLink()
l.failing = true
q, _ := aHaveQueue(l)
var told int
q.HaveWritten = func() { told++ }
q.SayFacts(context.Background(), appliedModules())
if told != 0 {
t.Fatalf("a refused write proved the build %d time(s)", told)
}
l.hmu.Lock()
l.failing = false
l.hmu.Unlock()
q.SayFacts(context.Background(), appliedModules())
if told == 0 {
t.Fatal("a have written under this build was not told")
}
// The bucket already holds what this node-engine would write: a restart with nothing changed still proves it.
same := newHaveLink()
q2, _ := aHaveQueue(same)
q2.SayFacts(context.Background(), appliedModules())
again, _ := aHaveQueue(same)
told = 0
again.HaveWritten = func() { told++ }
again.SayFacts(context.Background(), appliedModules())
if told == 0 {
t.Fatal("a have already saying the same was not told")
}
}
internal/link/health_test.go
+2 -2
View File
@@ -23,7 +23,7 @@ func (l *sayingLink) Health(_ context.Context, node string, body []byte) error {
}
func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
q := &Queue{Membership: Membership{Node: "anchor"}}
q := &Session{Membership: Membership{Node: "anchor"}}
h := Health{Contract: LivenessContract, At: time.Now().UTC(),
Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy, Reason: ReasonRestarting}}}
if q.SayHealth(t.Context(), h) {
@@ -77,7 +77,7 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
// the link was up, because the NATS link could not Ask). With no link open, and with a link that cannot
// ask, the check fails and says which.
func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
q := &Queue{Membership: Membership{Node: "anchor"}}
q := &Session{Membership: Membership{Node: "anchor"}}
if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
t.Fatalf("with no link open: %v", err)
}
internal/link/hearing.go
+5 -28
View File
@@ -8,9 +8,8 @@ import (
// What a host hears, as the host's own words for it.
//
// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and the
// declarations that arrive. The run loop reads its own words for a declaration rather than the
// client library's delivery type, so nothing past this file knows what carried it.
// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and what the controller asks of this
// node. A node hears no declaration: it follows its wants (novox/hq ADR 0539).
//
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005). This is its own
// interface over its own libraries, and it agrees with the controller only because a conformance
@@ -21,12 +20,9 @@ import (
// One interface rather than two, because dialling once is what keeps both halves of a node on the
// same connection.
type Link interface {
// Bus is what this node says: what it applied, and that it is here.
// Bus is what this node says: its facts, and that it is here.
Bus
// Declarations is what the mesh tells this node to be.
Declarations() <-chan Declaration
// Lost says the link ended, and why.
//
// **Read rather than discovered.** A node that finds out by noticing silence is a node that
@@ -38,31 +34,12 @@ type Link interface {
Close()
}
// Asked is a link that hears the mesh asking this node to say again what it last applied (novox/hq
// to-be 45 §6, the `report` verb). Optional: a link without it is never asked, and the mesh then sends
// the current declaration again instead.
// Asked is a link that hears the mesh asking this node to say its facts again. Optional: a link without it is never
// asked.
type Asked interface {
AskedToReport() <-chan struct{}
}
// Declaration is one thing the mesh told this node to be.
//
// **Handled, once — after the report is published.** A node that dies between applying and
// reporting leaves the declaration with the mesh and applies it again on return, which is safe
// because applying is reconciliation: it converges rather than repeating.
//
// There is one way of being done rather than two. A declaration set aside because a newer arrived
// with it is settled exactly as an applied one is, and the difference between them is a fact the
// *report* carries — a second method here would be a distinction the bus does not make.
type Declaration interface {
// Body is the signed declaration as it arrived, bytes unchanged: a node verifies what it
// received rather than what it re-encoded.
Body() []byte
// Handled settles it. Called after the report for it has been published, either way.
Handled() error
}
// Open opens this node's link to its mesh.
//
// Named Open rather than Dial because Dial is this package's raw TLS dial, which the enrolment path
internal/link/hearing_nats.go
+10 -86
View File
@@ -11,37 +11,16 @@ import (
"github.com/nats-io/nats.go"
)
// The host's link on the bus being built.
//
// Two things a host does here that it cannot do on the other bus, and one it must not try.
//
// **It declares nothing.** On the bus the mesh has, a host declares its own queue on connecting,
// because a queue that is not there means a node that hears nothing. Here the object it reads
// through is a durable consumer, and a host's account reaches no part of the JetStream API — by
// design, because the controller is the only writer of consumer definitions (design 25 §3). So the
// host **binds** to a consumer the controller made when this node enrolled, and a missing one is
// said as what it is rather than quietly created with whatever configuration this client happens to
// default to.
//
// **It gets order for free, and keeps the drain anyway.** The declaration subject is last-per-subject
// (design 29 §4), so a node that was away receives exactly the current declaration rather than a
// queue of superseded ones, and the stream's sequence orders them definitively — the wire-level
// answer to novox/hq issue 107. What the drain in run.go still answers is the live case: three
// pushes to a *connected* node are three deliveries whatever the stream later retains.
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts.
// EnrolSubject is where a joining machine asks. One subject for every node, because a machine
// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is
// the whole of what its enrolment user may do.
const EnrolSubject = "mesh.control.enrol"
// DeclareSubject is where this node's declaration lands. Its own, and no other node's: a host's
// account subscribes exactly this and the subject is the authority on which node a declaration is
// for.
func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" }
// AskReportSubject is where the mesh asks this node to say again what it last applied (novox/hq to-be 45
// §6): its own, on core NATS and off any stream. The answer is an ordinary report on its report subject,
// the one thing it already says — nobody's inbox is answered.
// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream.
// The answer is the facts event on its report subject — nobody's inbox is answered.
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
// natsURL is a bus address as the client wants it. A membership records host and port, because that
@@ -58,7 +37,6 @@ func natsURL(address string) string {
type natsLink struct {
conn *nats.Conn
js nats.JetStreamContext
sub *nats.Subscription
asking *nats.Subscription
asked chan struct{}
handing *nats.Subscription
@@ -72,9 +50,8 @@ type natsLink struct {
moving *nats.Subscription
node string
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
signer ed25519.PublicKey
arrived chan Declaration
lost chan error
signer ed25519.PublicKey
lost chan error
}
func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
@@ -123,27 +100,10 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e
l := &natsLink{
conn: conn, js: js, node: m.Node, signer: m.Signer,
arrived: make(chan Declaration, drainDepth),
lost: make(chan error, 1),
asked: make(chan struct{}, 1),
lost: make(chan error, 1),
asked: make(chan struct{}, 1),
}
// Bound to the consumer the controller made for this node, named after the node because that is
// what the node's own ack grant allows (`$JS.ACK.NODES.<node>.>`).
feed := make(chan *nats.Msg, drainDepth)
// The subject as well as the binding: the client checks what is asked for against the
// consumer's own filter, and an empty subject is refused rather than taken to mean "whatever
// that consumer delivers".
sub, err := js.ChanSubscribe(DeclareSubject(m.Node), feed, nats.Bind("NODES", m.Node))
if err != nil {
conn.Close()
return nil, fmt.Errorf(
"this node cannot read its declarations: %w. The mesh creates that when a node enrols, "+
"and a host may not create one itself — so this is the mesh's to answer, not this "+
"machine's", err)
}
l.sub = sub
l.hearAsks()
conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) {
@@ -159,29 +119,6 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e
}
})
go func() {
defer close(l.arrived)
for {
select {
case <-ctx.Done():
return
case msg, ok := <-feed:
if !ok {
select {
case l.lost <- errors.New("the bus stopped delivering"):
default:
}
return
}
select {
case l.arrived <- natsDeclaration{msg}:
case <-ctx.Done():
return
}
}
}
}()
return l, nil
}
@@ -299,14 +236,10 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
}
}
func (l *natsLink) Declarations() <-chan Declaration { return l.arrived }
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
func (l *natsLink) Lost() <-chan error { return l.lost }
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
func (l *natsLink) Lost() <-chan error { return l.lost }
func (l *natsLink) Close() {
if l.sub != nil {
_ = l.sub.Unsubscribe()
}
if l.asking != nil {
_ = l.asking.Unsubscribe()
}
@@ -375,12 +308,3 @@ var (
_ Asker = (*natsLink)(nil)
_ Asked = (*natsLink)(nil)
)
// natsDeclaration is one declaration off the NODES stream.
type natsDeclaration struct{ msg *nats.Msg }
func (d natsDeclaration) Body() []byte { return d.msg.Data }
// Handled acknowledges it. The ack goes to this node's own ack subject, which is the one thing
// besides its reports a node's account may publish.
func (d natsDeclaration) Handled() error { return d.msg.Ack() }
internal/link/hearing_nats_test.go
-337
View File
@@ -1,337 +0,0 @@
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"fmt"
"os"
"reflect"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// The host's link against a real server, because every claim here is about one.
//
// Whether binding to a consumer the host did not create works, whether a declaration on the node's
// own subject arrives, whether acknowledging it removes it from the consumer's pending — none of
// that can be reasoned out, and the first two are the ones that would leave a node silently hearing
// nothing:
//
// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNats
func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
t.Helper()
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
// **Ensured and purged, not deleted and recreated.** Delete-then-add looked like a reset and is
// not one: a test that did that inherited the previous test's messages, and the symptom was a
// declaration counted as delivered twice — which reads as a redelivery bug in the code under
// test rather than as a dirty stream. Purge is defined to empty a stream; recreating one is a
// race with the server's own teardown.
for _, want := range []*nats.StreamConfig{
{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, MaxMsgsPerSubject: 1},
{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
Retention: nats.WorkQueuePolicy},
} {
if _, err := js.StreamInfo(want.Name); err != nil {
if _, err := js.AddStream(want); err != nil {
t.Fatal(err)
}
}
if err := js.PurgeStream(want.Name); err != nil {
t.Fatal(err)
}
}
return conn, js
}
// theMeshMakes is the consumer the controller creates when a node enrols. Made here by the test
// because the host may not: its account reaches no part of the JetStream API, which is the whole
// reason this binds rather than subscribes.
//
// Removed afterwards, and each test names its own node: two tests sharing a consumer name share its
// delivery count and its pending list, and the first thing that goes wrong reads as a fault in the
// host rather than in the test beside it.
func theMeshMakes(t *testing.T, js nats.JetStreamContext, node string) {
t.Helper()
t.Cleanup(func() { _ = js.DeleteConsumer("NODES", node) })
if _, err := js.AddConsumer("NODES", &nats.ConsumerConfig{
Durable: node,
FilterSubject: DeclareSubject(node),
AckPolicy: nats.AckExplicitPolicy,
AckWait: 300 * time.Second,
DeliverSubject: "_DELIVER." + node,
}); err != nil {
t.Fatal(err)
}
}
func signedBy(t *testing.T, key ed25519.PrivateKey, declaration []byte) []byte {
t.Helper()
body, err := json.Marshal(Signed{
Declaration: declaration, Signature: ed25519.Sign(key, declaration),
})
if err != nil {
t.Fatal(err)
}
return body
}
// A declaration on this node's own subject reaches the host, is applied, and acknowledging it
// empties the consumer — which is what tells the mesh the node has it.
func TestNatsADeclarationReachesTheHostAndIsSettled(t *testing.T) {
conn, js := aBus(t)
const node = "settling"
theMeshMakes(t, js, node)
public, private, _ := ed25519.GenerateKey(nil)
m := Membership{Node: node, Signer: public}
// Dialled directly rather than through Open: the test server has no TLS, and what is being
// checked is the subscription and the settling, not the pin — which PinnedConfig owns and its
// own tests cover.
l := &natsLink{conn: conn, js: js, node: node,
arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)}
feed := make(chan *nats.Msg, drainDepth)
sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node))
if err != nil {
t.Fatalf("the host could not bind to the consumer the mesh made for it: %v", err)
}
defer func() { _ = sub.Unsubscribe() }()
go func() {
for msg := range feed {
l.arrived <- natsDeclaration{msg}
}
}()
if _, err := js.Publish(DeclareSubject(node),
signedBy(t, private, []byte(`{"declared":"d1"}`))); err != nil {
t.Fatal(err)
}
select {
case d := <-l.Declarations():
report := handleBody(context.Background(), m, d.Body(),
func(context.Context, []byte, []byte) Report {
return Report{Applied: []string{"store"}}
})
if report.Refused != "" {
t.Fatalf("a declaration the mesh signed was refused: %s", report.Refused)
}
if err := d.Handled(); err != nil {
t.Fatalf("the node could not acknowledge its own declaration: %v", err)
}
case <-time.After(8 * time.Second):
t.Fatal("no declaration reached the host")
}
// **Nothing pending is the property**; a delivery count is not. Delivery is at-least-once by
// design, so pinning "delivered exactly once" would be asserting something the mesh does not
// rely on. What matters is that the acknowledgement landed, so the mesh can tell the node has
// it — and that no redelivery was needed to get there, which is what would say the node was
// too slow to answer for its own ack wait.
deadline := time.Now().Add(5 * time.Second)
var last string
for time.Now().Before(deadline) {
info, err := js.ConsumerInfo("NODES", node)
switch {
case err != nil:
last = err.Error()
case info.NumAckPending == 0 && info.NumRedelivered == 0:
return
default:
last = fmt.Sprintf("pending %d, redelivered %d", info.NumAckPending, info.NumRedelivered)
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("the declaration was not settled, so the mesh cannot tell the node has it: %s", last)
}
// **A node that was away gets exactly the current declaration and nothing older.** Three pushed
// while nothing is listening leave one on the stream, and it is the newest — the wire-level answer
// to novox/hq issue 107, and the half of the drain that stops being the host's problem.
func TestNatsANodeThatWasAwayGetsOnlyTheNewest(t *testing.T) {
_, js := aBus(t)
const node = "returning"
_, private, _ := ed25519.GenerateKey(nil)
for _, id := range []string{"d1", "d2", "d3"} {
if _, err := js.Publish(DeclareSubject(node),
signedBy(t, private, []byte(`{"declared":"`+id+`"}`))); err != nil {
t.Fatal(err)
}
}
info, err := js.StreamInfo("NODES")
if err != nil {
t.Fatal(err)
}
if info.State.Msgs != 1 {
t.Fatalf("%d declarations survived for one node; a node that was away would apply a backlog "+
"of things nobody wants any more", info.State.Msgs)
}
theMeshMakes(t, js, node)
feed := make(chan *nats.Msg, drainDepth)
sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node))
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
select {
case msg := <-feed:
if want := declaredIn(signedBy(t, private, []byte(`{"declared":"d3"}`))); declaredIn(msg.Data) != want {
t.Fatalf("the node was given %q rather than the newest", msg.Data)
}
case <-time.After(8 * time.Second):
t.Fatal("the node that was away was given nothing")
}
}
// A report goes through the stream: it must survive the controller's store restarting.
func TestNatsAReportIsKept(t *testing.T) {
conn, js := aBus(t)
bus := OverNATS{Conn: conn, JS: js}
ctx := context.Background()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1"})
if err := bus.Report(ctx, "anchor", body); err != nil {
t.Fatal(err)
}
info, err := js.StreamInfo("CONTROL")
if err != nil {
t.Fatal(err)
}
if info.State.Msgs != 1 {
t.Fatalf("%d messages were kept; a report must be, once", info.State.Msgs)
}
}
// **One apply queue, against a real bus** (novox/hq to-be 45 §6, R2). A declaration is being applied
// when two more are pushed and the five-minute reconcile comes due: the worker applies the one in
// hand, then only the newest — the reconcile is that apply — and the mesh's stream holds one account
// per apply, each naming the order it applied, numbered in the order they were made. Every
// declaration delivered is settled.
func TestNatsTheQueueAppliesTheNewestOnceAndReportsInOrder(t *testing.T) {
conn, js := aBus(t)
const node = "queueing"
theMeshMakes(t, js, node)
public, private, _ := ed25519.GenerateKey(nil)
m := Membership{Node: node, Signer: public}
ctx, stop := context.WithCancel(context.Background())
defer stop()
l := &natsLink{conn: conn, js: js, node: node,
arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)}
feed := make(chan *nats.Msg, drainDepth)
sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node))
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
go func() {
for msg := range feed {
l.arrived <- natsDeclaration{msg}
}
}()
reports, err := js.SubscribeSync(ReportSubject(node), nats.BindStream("CONTROL"))
if err != nil {
t.Fatal(err)
}
defer func() { _ = reports.Unsubscribe() }()
a := &applies{}
inFirst, release := make(chan struct{}, 1), make(chan struct{})
reconciled := 0
q := &Queue{Membership: m, Timeout: 5 * time.Second, Unsaid: &keptInMemory{},
Apply: func(ctx context.Context, raw, sig []byte) Report {
r := a.apply(ctx, raw, sig)
if r.Sequence == 1 {
inFirst <- struct{}{}
<-release
}
return r
},
Reconcile: func(context.Context) (Report, bool) { reconciled++; return Report{}, false },
}
worker := runWorker(ctx, q)
go func() { _ = serve(ctx, l, m, q, nil, 5*time.Second) }()
push := func(id string, sequence int64) {
inner, _ := json.Marshal(map[string]any{"declaration": 1, "epoch": 7, "sequence": sequence,
"resources": []any{map[string]any{"id": id}}})
if _, err := js.Publish(DeclareSubject(node), signedBy(t, private, inner)); err != nil {
t.Fatal(err)
}
}
push("one", 1)
select {
case <-inFirst:
case <-time.After(8 * time.Second):
t.Fatal("the first declaration never reached the worker")
}
push("two", 2)
push("three", 3)
q.ReconcileDue()
time.Sleep(2 * drainWindow) // the link gathers both and hands them to the queue
close(release)
var heard []Report
for len(accounts(heard)) < 2 {
msg, err := reports.NextMsg(8 * time.Second)
if err != nil {
t.Fatalf("the mesh heard %+v and then nothing: %v", heard, err)
}
var r Report
if err := json.Unmarshal(msg.Data, &r); err != nil {
t.Fatal(err)
}
heard = append(heard, r)
_ = msg.Ack()
}
stop()
<-worker
if got := a.all(); !reflect.DeepEqual(got, []string{"one", "three"}) {
t.Fatalf("applied %v; the one in hand and then only the newest were wanted", got)
}
if reconciled != 0 {
t.Fatal("the reconcile applied beside the delivery it was due with")
}
accounted := accounts(heard)
if accounted[0].Sequence != 1 || accounted[1].Sequence != 3 || accounted[1].Epoch != 7 {
t.Fatalf("the accounts do not name what was applied: %+v", accounted)
}
for i := 1; i < len(heard); i++ {
if heard[i].ReportSequence <= heard[i-1].ReportSequence {
t.Fatalf("the reports left out of the order they were made: %+v", heard)
}
}
deadline := time.Now().Add(5 * time.Second)
for {
info, err := js.ConsumerInfo("NODES", node)
if err == nil && info.NumAckPending == 0 {
break
}
if time.Now().After(deadline) {
t.Fatalf("a delivered declaration was left unsettled: %+v %v", info, err)
}
time.Sleep(20 * time.Millisecond)
}
}
internal/link/held_note_test.go
-46
View File
@@ -1,46 +0,0 @@
package link
import (
"strings"
"testing"
)
// The count that did not add up was the only symptom sixteen held resources had, and reading it meant
// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did
// says what it did not, too.
func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) {
got := heldNote([]Held{
{ID: "ca", Module: "route-proxy", Kind: "directory"},
{ID: "certs", Module: "route-proxy", Kind: "directory"},
{ID: "server", Module: "route-proxy", Kind: "container"},
{ID: "mail", Module: "mailu", Kind: "container"},
})
if !strings.Contains(got, "4 held") {
t.Fatalf("the count of what was held is not in the line: %q", got)
}
// The module is the thing an operator can act on: `take` takes a module.
if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") {
t.Fatalf("the line does not break the holds down by module: %q", got)
}
// Ordered, so two machines holding the same things read the same and a diff of two reports is
// about what changed.
if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") {
t.Fatalf("modules are not in a stated order: %q", got)
}
// It says why, because "held" alone reads as a failure and this is correct behaviour.
if !strings.Contains(got, "taken") {
t.Fatalf("the line does not say a hold ends when the module is taken: %q", got)
}
}
func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) {
// A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of
// them is how a line stops being read.
if got := heldNote(nil); got != "" {
t.Fatalf("an apply with no holds added %q to its line", got)
}
if got := heldNote([]Held{}); got != "" {
t.Fatalf("an apply with no holds added %q to its line", got)
}
}
internal/link/messages.go
+8 -126
View File
@@ -18,22 +18,6 @@ const (
KeyReport = "report"
)
// Signed is a declaration and the signature over it.
//
// novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is
// verified by its signature, every time**. The two are different questions — a node connects to
// the broker and takes instruction from the control plane behind it, and pinning only the first
// would make the second transitive.
//
// The signature is over Declaration exactly as it arrived, bytes unchanged. Re-encoding before
// verifying would mean checking a signature over something other than what was sent, and any
// difference in key order or spacing would break it — so the raw message is what is signed and
// what is checked.
type Signed struct {
Declaration []byte `json:"declaration"`
Signature []byte `json:"signature"`
}
// Report is what a node says after applying, and it is a statement rather than a write.
//
// A node states; the context that owns the data writes (novox/hq ADR 0006). The difference is the
@@ -51,14 +35,6 @@ type Report struct {
// Refused is set when the declaration was rejected whole rather than applied in part.
Refused string `json:"refused,omitempty"`
// Superseded names the newer declaration this one was set aside for, unapplied.
//
// A machine asked to be five successive things becomes the last one (novox/hq issue 031):
// when several declarations are waiting, the host applies the newest and acknowledges the
// rest without applying them. Each of those is still reported, because silence reads as a
// machine that ignored an instruction and "applied" would be a lie — this is the third word.
Superseded string `json:"superseded,omitempty"`
// Carried are the machine's ports held by what this host raised from its own bundle.
//
// **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node
@@ -70,62 +46,13 @@ type Report struct {
// machine saying what is true of it, not asking for anything.
Carried []int `json:"carried,omitempty"`
// Declared is the digest of the declaration this report is about — sha256 of the exact bytes
// the mesh sent, which the mesh recorded when it sent them.
//
// **Which declaration, not when.** The mesh compared its send time to this report's arrival
// to decide whether a machine had caught up, and lost the race it invited: an apply started
// under the previous declaration finishes after the next one is sent, its report lands newer
// than the send, and the machine reads as caught up with words it has not read yet. Clocks
// cannot answer "which"; the digest is the answer itself.
Declared string `json:"declared,omitempty"`
// Order is where the declaration this report is about stands — its `epoch` and `sequence`, as
// the declaration carried them — so the mesh orders reports by what they are about rather than
// by when they arrived (novox/hq to-be 45 §6). On the wire as two top-level keys beside
// `declared`; absent for a declaration that claimed no order, and for a report about none.
// Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand
// among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine,
// the sequence one higher for every facts event, so the controller refuses older facts by this order. On the
// wire as three top-level keys.
Writer string `json:"writer,omitempty"`
Order
// ReportSequence is this node-engine's own order among everything it reports: one higher for
// every report it makes, kept on disk so it goes on increasing across restarts and self-updates
// (to-be 45 §6). With the declaration's order it lets the mesh refuse an older report about the
// same declaration — a reconcile's account overtaking the apply that followed it (novox/hq issue
// 267) is then refused by number rather than set aside by digest. A report said again because
// it never reached the mesh (issue 264) carries the number it was made with.
//
// Zero claims no order: every report an older host makes, and a one-shot report a command makes
// (a rekey). **Its presence also says this host reads a declaration's `epoch`**, which is how the
// mesh knows it may send one: an older host decodes a declaration strictly and refuses a key it
// does not know.
ReportSequence int64 `json:"report_sequence,omitempty"`
// OlderThan is set on a report refusing a declaration older than one this node has applied: the
// order of the newer declaration it holds (to-be 45 §6, rule 2). The refused declaration is the
// report's own Declared and Order, and Refused says it in words. Applying it would make the
// machine into something a controller that lost its lease said, after one that holds it.
OlderThan *Order `json:"older_than,omitempty"`
// RefusedOlder is how many declarations this node-engine has refused as older, ever — kept on
// disk with the report sequence. On every report, so a count missed with a lost refusal is still
// read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts.
RefusedOlder int64 `json:"refused_older,omitempty"`
// ReadsGeneration says this node-engine reads a declaration's assignment generation and its
// put-back mark (novox/hq issue 234), on every report it makes: the controller sends them only to a
// machine that said so, because an older node-engine decodes strictly and refuses a key it does not
// know, whole. A node-engine rolled back says so no longer.
ReadsGeneration bool `json:"reads_generation,omitempty"`
// ReadsModules says this node-engine reads a resource's `module` and a declaration's `builds` (novox/hq ADR 0429
// §4, issue 1132), on every report it makes, beside ReadsGeneration: the controller sends them only to a machine
// that said so, for the same reason.
ReadsModules bool `json:"reads_modules,omitempty"`
// OlderGeneration is set on a report refusing a declaration composed from an older assignment
// generation than this machine applied (novox/hq issue 234). The refused declaration is the report's
// own Declared and Order, so the controller can name who sent it from its record of sends.
OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"`
// Held is what this adopted node found and is keeping as it was until its module is taken
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
Held []Held `json:"held,omitempty"`
@@ -647,52 +574,14 @@ type Rollback struct {
At time.Time `json:"at"`
}
// Order is where a declaration stands among everything the mesh has sent this node (novox/hq to-be
// 45 §6): the controller's lease **epoch** it was sent under, and its **sequence** — one higher for
// every send to this node (04-ISSUES/107). The declaration carries both inside what is signed, as
// top-level `epoch` and `sequence` beside `declaration`; a report carries back the pair of the
// declaration it is about.
//
// Zero in either is "no order claimed", never "first": every declaration an older controller sent,
// and the bundle genesis applies.
// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its
// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one
// with nothing kept on disk.
type Order struct {
Epoch int64 `json:"epoch,omitempty"`
Sequence int64 `json:"sequence,omitempty"`
}
// Older says whether a declaration of this order is older than one of order `than`, which this node
// has applied — the one rule by which a node-engine refuses a declaration (to-be 45 §6: "older epoch;
// same epoch, lower sequence").
//
// **Only when both claim an epoch.** A declaration with none is an older controller's — or a
// controller rolled back to a build from before the lease — and one kept with none is what this host
// held before any controller had a lease. Neither has an order to compare, and refusing on a guess
// would strand the machine the moment the controller is older than the host: without an epoch, today's
// behaviour stands. A higher epoch is a new lease holder and is never older, whatever its sequence.
func (o Order) Older(than Order) bool {
if o.Epoch <= 0 || than.Epoch <= 0 {
return false
}
if o.Epoch != than.Epoch {
return o.Epoch < than.Epoch
}
return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence
}
// Supersedes says whether a declaration of order `o`, arriving after one of order `before`, takes its
// place among what is waiting to be applied. By epoch when both claim one and they differ, then by
// sequence when both claim one, and by arrival when they do not — what the drain had to go on before
// declarations said where they stand (04-ISSUES/107).
func (o Order) Supersedes(before Order) bool {
if o.Epoch > 0 && before.Epoch > 0 && o.Epoch != before.Epoch {
return o.Epoch > before.Epoch
}
if o.Sequence > 0 && before.Sequence > 0 {
return o.Sequence >= before.Sequence
}
return true
}
// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a
// Stringer promoted onto every report would print each one as its order alone.
func (o Order) Words() string {
@@ -842,13 +731,6 @@ type FoundFirewall struct {
RetiredBy string `json:"retired_by,omitempty"`
}
// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that
// generation, and the highest this machine applied (novox/hq issue 234).
type GenerationRefusal struct {
Generation int64 `json:"generation"`
Applied int64 `json:"applied"`
}
// ModuleFact is how one module's part of an apply came out: the build the node was sent for it, and whether every
// resource of it applied. Detail names the failing resource and its error (issue 1132).
type ModuleFact struct {
internal/link/messages_test.go
+3 -106
View File
@@ -1,115 +1,10 @@
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"testing"
)
// verified runs what Run does to a delivery body, without a broker: unmarshal, check the
// signature, and only then apply. Isolating it keeps this test about the check rather than about
// the bus, which is tested against a real one in the lab.
func verified(t *testing.T, signer ed25519.PublicKey, body []byte) (Report, bool) {
t.Helper()
applied := false
report := handleBody(context.Background(), Membership{Node: "anchor", Signer: signer}, body,
func(context.Context, []byte, []byte) Report {
applied = true
return Report{Applied: []string{"something"}}
})
return report, applied
}
func signedBody(t *testing.T, private ed25519.PrivateKey, declaration string) []byte {
t.Helper()
raw, err := json.Marshal(Signed{
Declaration: []byte(declaration),
Signature: ed25519.Sign(private, []byte(declaration)),
})
if err != nil {
t.Fatal(err)
}
return raw
}
func TestTheMeshsOwnDeclarationIsApplied(t *testing.T) {
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
report, applied := verified(t, public, signedBody(t, private, `{"declaration":1}`))
if !applied {
t.Fatalf("a declaration the mesh signed was not applied: %s", report.Refused)
}
}
// Defends novox/hq ADR 0002: everything reaching a node arrives over the broker — and therefore
// ADR 0004's consequence, that the broker is not trusted to say who is speaking.
//
// A transport nobody authenticates per-message would let whatever holds the connection attribute
// a declaration to any node it liked.
func TestAForgedDeclarationIsNeverApplied(t *testing.T) {
// The check that stands between "the mesh changes this machine" and "anybody does". The host
// applies whatever the link delivers, so a forged declaration is the whole machine.
public, _, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
_, other, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
report, applied := verified(t, public, signedBody(t, other, `{"declaration":1}`))
if applied {
t.Fatal("a declaration signed by another key was applied")
}
if report.Refused != ErrForged.Error() {
t.Errorf("refused, but not as a forgery: %q", report.Refused)
}
}
func TestATamperedDeclarationIsNeverApplied(t *testing.T) {
// A broker that changed the declaration in flight, keeping the signature. This is what makes
// pinning the transport insufficient on its own.
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(Signed{
Declaration: []byte(`{"declaration":1,"resources":["something else entirely"]}`),
Signature: ed25519.Sign(private, []byte(`{"declaration":1}`)),
})
if err != nil {
t.Fatal(err)
}
report, applied := verified(t, public, raw)
if applied {
t.Fatal("a declaration altered after signing was applied")
}
if report.Refused != ErrForged.Error() {
t.Errorf("refused, but not as a forgery: %q", report.Refused)
}
}
func TestAMalformedMessageIsToldApartFromAForgery(t *testing.T) {
// novox/hq ADR 0004 requires these to be distinguishable: one means somebody is trying, the
// other means something is broken, and they need different responses from a person.
public, _, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
report, applied := verified(t, public, []byte("this is not a message"))
if applied {
t.Fatal("something unparseable was applied")
}
if report.Refused == ErrForged.Error() {
t.Error("a malformed message was reported as a forgery; those must be distinguishable")
}
}
func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
// The contract with the control plane, which defines these separately. A matching test lives
// there; rename a field on either side and both fail.
@@ -117,7 +12,9 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
value any
expect []string
}{
{Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}},
// novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence.
{Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}},
[]string{"node", "writer", "epoch", "sequence"}},
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
[]string{"node", "applied", "failed", "refused"}},
// novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what
internal/link/movesecret.go
+1 -1
View File
@@ -173,7 +173,7 @@ const VaultWithin = 15 * time.Second
// AskVault asks one of the vault's verbs in this node's engine's own name over the link open now, and answers its
// result, or its refusal as an error.
func (q *Queue) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) {
func (q *Session) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) {
q.init()
q.mu.Lock()
bus := q.bus
internal/link/movesecret_test.go
+2 -2
View File
@@ -112,7 +112,7 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) {
}
defer vault.Unsubscribe()
l := &natsLink{conn: conn, js: js, node: m.Node, signer: m.Signer}
q := &Queue{Membership: m}
q := &Session{Membership: m}
q.attach(t.Context(), l)
l.HearMoveSecrets(func(s MoveSecret) (MovedSecret, error) {
raw, err := q.AskVault(t.Context(), "move-in", map[string]string{"module": s.Module})
@@ -135,7 +135,7 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) {
if err := json.Unmarshal(reply.Data, &said); err != nil || said.Secret != "own/"+m.Node+"/postgres/superuser-password" {
t.Fatalf("answered %s, %v", reply.Data, err)
}
if _, err := (&Queue{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) {
if _, err := (&Session{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) {
t.Fatalf("a queue with no link asked the vault: %v", err)
}
}
internal/link/newest_test.go
-79
View File
@@ -1,79 +0,0 @@
package link
import (
"sync"
"testing"
"time"
)
type said struct {
body []byte
mu sync.Mutex
handled bool
}
func (s *said) Body() []byte { return s.body }
func (s *said) Handled() error {
s.mu.Lock()
defer s.mu.Unlock()
s.handled = true
return nil
}
func (s *said) wasHandled() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.handled
}
func arriving(bodies ...string) chan Declaration {
ch := make(chan Declaration, 8)
for _, b := range bodies {
ch <- &said{body: []byte(b)}
}
return ch
}
// newest is what the queue's worker would take from what the link gathered behind `first`.
func newest(arriving <-chan Declaration, first Declaration, window time.Duration) (Declaration, []Declaration) {
return pick(gather(arriving, first, window))
}
// A machine asked to be five things becomes the last one: what is already waiting supersedes what
// arrived first, and everything set aside is named so it can be reported.
func TestWhatIsAlreadyWaitingSupersedesWhatArrivedFirst(t *testing.T) {
waiting := arriving("two", "three", "four")
apply, superseded := newest(waiting, &said{body: []byte("one")}, 50*time.Millisecond)
if string(apply.Body()) != "four" {
t.Fatalf("applied %q, not the newest", apply.Body())
}
if len(superseded) != 3 || string(superseded[0].Body()) != "one" ||
string(superseded[2].Body()) != "three" {
t.Fatalf("set aside %d: %v", len(superseded), superseded)
}
}
// One declaration with nothing behind it is applied as it always was, after the window.
func TestALoneDeclarationIsAppliedAfterTheWindow(t *testing.T) {
began := time.Now()
apply, superseded := newest(arriving(), &said{body: []byte("only")}, 30*time.Millisecond)
if string(apply.Body()) != "only" || len(superseded) != 0 {
t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded))
}
if time.Since(began) < 30*time.Millisecond {
t.Fatal("did not wait the window for a straggler")
}
}
// A straggler within the window is taken; one after it is the next push.
func TestAStragglerWithinTheWindowIsTaken(t *testing.T) {
waiting := arriving()
go func() {
time.Sleep(20 * time.Millisecond)
waiting <- &said{body: []byte("late")}
}()
apply, superseded := newest(waiting, &said{body: []byte("first")}, 100*time.Millisecond)
if string(apply.Body()) != "late" || len(superseded) != 1 {
t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded))
}
}
internal/link/nothing_alive_test.go
+1 -1
View File
@@ -35,7 +35,7 @@ func (l *listeningForAlive) count() int {
func TestTheLinkLoopPublishesNoAlive(t *testing.T) {
m, _ := aMember(t)
l := &listeningForAlive{quietLink: newQuietLink()}
q := aQueue(m, nil, &keptInMemory{})
q := &Session{Membership: m, Timeout: time.Second}
ctx, stop := context.WithCancel(context.Background())
done := make(chan error, 1)
internal/link/order_test.go
-50
View File
@@ -1,50 +0,0 @@
package link
import (
"encoding/json"
"testing"
"time"
)
// The drain picked the last to arrive. A backlog longer than the batch, or a broker that split a
// burst, delivered a superseded declaration last (novox/hq 04-ISSUES/107).
func sequenced(t *testing.T, n int64) *said {
t.Helper()
inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{}, "sequence": n})
if err != nil {
t.Fatal(err)
}
body, err := json.Marshal(Signed{Declaration: inner, Signature: []byte("s")})
if err != nil {
t.Fatal(err)
}
return &said{body: body}
}
func TestTheDrainKeepsTheHighestSequenceNotTheLastToArrive(t *testing.T) {
waiting := make(chan Declaration, 8)
waiting <- sequenced(t, 9)
waiting <- sequenced(t, 4) // arrived last, composed earlier
latest, aside := newest(waiting, sequenced(t, 8), 30*time.Millisecond)
if got := orderOf(latest.Body()).Sequence; got != 9 {
t.Fatalf("the drain kept sequence %d, and 9 was waiting", got)
}
if len(aside) != 2 {
t.Fatalf("%d set aside, wanted 2 (the 8 and the late 4)", len(aside))
}
}
func TestWithoutSequencesTheLastToArriveStillWins(t *testing.T) {
// The behaviour this had before, kept for a controller that sends no order.
apply, aside := newest(arriving("two", "three"), &said{body: []byte("one")}, 30*time.Millisecond)
if string(apply.Body()) != "three" || len(aside) != 2 {
t.Fatalf("applied %q with %d set aside", apply.Body(), len(aside))
}
}
func TestAnUnreadableBodyClaimsNoOrder(t *testing.T) {
if got := orderOf([]byte("not json")); got != (Order{}) {
t.Fatalf("garbage claimed %+v", got)
}
}
internal/link/queue.go
-647
View File
@@ -1,647 +0,0 @@
package link
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1).
//
// **One worker applies; everything else asks it to.** A declaration delivered over the link and the
// five-minute reconcile are two reasons to *enqueue* the same act, never two paths that apply. Before
// this they were two, each taking an apply lock, and every order between them was a fault met on the
// live mesh: the reconcile read what was kept before a delivery and applied it over the newer one
// (novox/hq issues 257, 261); the reconcile went first and its report about the older declaration
// reached the mesh after the delivery's (267). Each fix closed one order and left the class open.
//
// The queue holds at most one pending request, coalesced: whatever has been delivered and not yet
// taken, and whether a reconcile is due. When the worker starts it takes the **newest declaration held
// at that moment**, sets the rest aside — each reported as set aside, then settled unapplied — applies
// it once, and makes **one report**, naming the declaration's order and its own report sequence. A
// reconcile due while a delivery waits is that delivery's apply: applying the newest thing the mesh
// said is what reconciling is. Only with nothing delivered does a reconcile apply what was kept, read
// once its turn has come.
//
// **Reports leave in the order they are made**, because the worker makes and says them, one at a time.
// A report made while the link is down waits: an apply's in the unsaid store (issue 264), a
// reconcile's in one slot replaced by anything newer, and both are said, in that order, when the next
// link opens — before anything newly delivered is applied.
type Queue struct {
// Membership is whose declarations these are: their signer, and the node the reports are from.
Membership Membership
// Apply applies a declaration the mesh signed.
Apply Applier
// Reconcile holds the machine to what it was last told, read once it is this act's turn. False
// when there is nothing to hold it to, or the reconcile could not run; it says why itself.
Reconcile Reconcile
// News is whether a reconcile's report is worth saying unasked (novox/hq ADR 0100, ADR 0140). Nil
// is never: a reconcile is otherwise silent.
News func(Report) bool
// Heard is told every account of the machine the mesh has taken — an apply's or a reconcile's,
// not a refusal or a declaration set aside, which say nothing about the machine. Nil is allowed.
Heard func(Report)
// Unsaid keeps an apply's report until the mesh has taken it (issue 264). Nil keeps nothing.
Unsaid Unsaid
// Numbers keeps the report sequence and the refusals counted, across restarts and self-updates.
// Nil counts in memory, which is only right where nothing outlives the process — a test.
Numbers Numbers
// HandOver judges and records a directory used as found handed to the mesh by the operator, asked by
// the controller over the link (novox/hq issue 356). Nil answers every such ask with a refusal.
HandOver HandOverFunc
// SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the
// controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal.
SetuidSearch SetuidSearchFunc
// JudgeDirectories judges this node's declared directories read-only, asked by the controller over the
// link (novox/hq issue 358). Nil answers every such ask with a refusal.
JudgeDirectories JudgeDirectoriesFunc
// AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved
// warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal.
AccessAct AccessActFunc
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
MoveSecret MoveSecretFunc
Say Announce
Timeout time.Duration
once sync.Once
wake chan struct{}
mu sync.Mutex
idle *sync.Cond
// delivering is a delivered declaration in hand: its report is owed on the link it came over, so
// that link is not let go until it is said (detach). A reconcile in hand holds no link — its report
// waits for the next one if this one goes.
delivering bool
waiting []Declaration // delivered and not yet taken, in the order they arrived
due bool // a reconcile asked for
// asked is the mesh asking this node to say what it last applied (the `report` verb, to-be 45 §6):
// the reconcile it enqueues says its report whether or not it is news.
asked bool
bus Bus // the link open now; nil while there is none
unasked *Report // a reconcile's report not yet said, made while there was no link
// saying is held while a report is made and said, so the order they leave in is the order they
// are made in — whoever says them, the worker or a link opening.
saying sync.Mutex
counted bool
numbers numbered
// haves is what this node-engine writes beside its reports (novox/hq ADR 0429 §4, issue 1132).
haves haveState
}
// Reconcile is the reconcile's act, run by the queue's worker. See Queue.Reconcile.
type Reconcile func(ctx context.Context) (Report, bool)
// Numbers is where the queue keeps what it counts, so a successor goes on from where it stopped.
type Numbers interface {
// Read is what was kept: zero for a node that never reported, an error when it cannot be read —
// never zero for "could not tell".
Read() (reportSequence, refusedOlder int64, err error)
// Save keeps them, on disk when it returns.
Save(reportSequence, refusedOlder int64) error
}
type numbered struct{ sequence, refused int64 }
func (q *Queue) init() {
q.once.Do(func() {
q.wake = make(chan struct{}, 1)
q.idle = sync.NewCond(&q.mu)
if q.Say == nil {
q.Say = func(string) {}
}
})
}
// Deliver enqueues what the link delivered, in the order it arrived. It never waits for an apply.
func (q *Queue) Deliver(arrived ...Declaration) {
q.init()
q.mu.Lock()
q.waiting = append(q.waiting, arrived...)
q.mu.Unlock()
q.signal()
}
// ReconcileDue enqueues a reconcile. One asked for while another is waiting is the same one.
func (q *Queue) ReconcileDue() {
q.init()
q.mu.Lock()
q.due = true
q.mu.Unlock()
q.signal()
}
// ReportAsked is the mesh asking this node to say again what it last applied (novox/hq to-be 45 §6, the
// `report` verb its healer H1 asks when a send went unreported). **An enqueue, like everything else**: a
// reconcile, whose report — the account of the declaration this node keeps, the last it applied — is
// said whether or not it is news. A delivery waiting meanwhile is applied instead and reported, which
// answers the question better. Asked again before the worker takes it is the same question.
func (q *Queue) ReportAsked() {
q.init()
q.mu.Lock()
q.due, q.asked = true, true
q.mu.Unlock()
q.signal()
}
func (q *Queue) signal() {
select {
case q.wake <- struct{}{}:
default:
// One is already waiting to be read, and the worker takes everything pending when it does.
}
}
// Run is the worker. It returns once the context ends and the act in hand is finished and said:
// asked to stop — the host standing aside for its successor — nothing further is applied.
func (q *Queue) Run(ctx context.Context) {
q.init()
for {
for q.step(ctx) {
}
select {
case <-ctx.Done():
return
case <-q.wake:
}
}
}
// step takes what is pending and does it, once. False when there was nothing to do, or the queue
// was asked to stop.
func (q *Queue) step(ctx context.Context) bool {
q.mu.Lock()
if ctx.Err() != nil || (len(q.waiting) == 0 && !q.due) {
q.mu.Unlock()
return false
}
batch, due, asked := q.waiting, q.due, q.asked
q.waiting, q.due, q.asked = nil, false, false
q.delivering = len(batch) > 0
q.mu.Unlock()
defer func() {
q.mu.Lock()
q.delivering = false
q.idle.Broadcast()
q.mu.Unlock()
}()
if len(batch) == 0 {
q.reconcile(ctx, asked)
return true
}
latest, aside := pick(batch)
for _, old := range aside {
if d := declaredIn(old.Body()); d != "" && d == declaredIn(latest.Body()) {
// The same declaration again — delivered twice while the worker was busy. Nothing is set
// aside: it is about to be applied.
_ = old.Handled()
continue
}
q.Say(fmt.Sprintf("set aside declaration %s (%s): a newer one arrived with it",
short(declaredIn(old.Body())), orderOf(old.Body()).Words()))
q.tell(ctx, Report{Declared: declaredIn(old.Body()), Order: orderOf(old.Body()),
Superseded: declaredIn(latest.Body())}, setAside, old)
}
report := handleBody(q.withFiles(ctx), q.Membership, latest.Body(), q.Apply)
switch {
case report.OlderThan != nil:
// **Refused, counted and said** (rule 2): one line here, the count on this and every later
// report, and the report itself naming what was refused and what is held.
total := q.countRefusal()
q.Say(fmt.Sprintf("refused declaration %s (%s): this node applied %s, from a newer lease "+
"holder — %d refused as older so far", short(report.Declared), report.Order.Words(),
report.OlderThan.Words(), total))
case report.OlderGeneration != nil:
// Said here, and the report names it, so the controller raises it naming the sender (novox/hq
// issue 234). Not counted with the lease's refusals: those are S13's, by epoch.
q.Say(fmt.Sprintf("refused declaration %s (%s): it was composed from assignment generation %d, and "+
"this node applied generation %d", short(report.Declared), report.Order.Words(),
report.OlderGeneration.Generation, report.OlderGeneration.Applied))
case report.Refused != "":
q.Say("refused a declaration: " + report.Refused)
case len(report.Failed) > 0:
q.Say(fmt.Sprintf("applied %d and failed: %v%s",
len(report.Applied), report.Failed, heldNote(report.Held)))
default:
q.Say(fmt.Sprintf("applied %d resource(s)%s", len(report.Applied), heldNote(report.Held)))
}
if due && report.Refused != "" {
// Nothing was applied, so the reconcile this stood in for has not happened. It runs next.
q.mu.Lock()
q.due = true
q.mu.Unlock()
}
// A refusal as older is not kept to be said again: what the mesh waits for from this node is the
// account of the last apply, and that is still what is kept.
kind := applied
if report.OlderThan != nil || report.OlderGeneration != nil {
kind = refusedOlder
}
q.tell(ctx, report, kind, latest)
return true
}
func (q *Queue) reconcile(ctx context.Context, asked bool) {
if q.Reconcile == nil {
if asked {
q.Say("asked to say what this node last applied, and it holds nothing to reconcile with")
}
return
}
report, ok := q.Reconcile(q.withFiles(ctx))
if !ok {
if asked {
q.Say("asked to say what this node last applied, and the reconcile could not say it")
}
return
}
if !asked && (q.News == nil || !q.News(report)) {
// Not worth a report, but a reconcile found how each module is now: the have keys say it (issue 1132), so a
// node-engine that restarted writes them again without waiting for a change.
if q.noteApply(report) {
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
q.writeHaves(ctx, bus)
}
return
}
if asked {
q.Say(fmt.Sprintf("asked by the mesh, saying again what this node last applied: declaration %s (%s)",
short(report.Declared), report.Order.Words()))
}
q.tell(ctx, report, unasked)
}
// What a report is, for what tell does with it besides saying it.
type kindOf int
const (
// applied is an apply's account: kept until said (issue 264), and fresher than any reconcile's
// report still waiting.
applied kindOf = iota
// setAside is a declaration a newer one took the place of, unapplied.
setAside
// refusedOlder is a declaration refused as older than what was applied. Not kept: what the mesh
// waits for from this node is still the account of the last apply.
refusedOlder
// unasked is a reconcile's report. Not said — no link, or the bus refused it — the newest waits
// for the next link, and anything the worker says before then replaces it.
unasked
)
// tell makes a report this node's — its node, its report sequence, the refusals counted — and says it
// on the link open now. Kept first, when it is an apply's, so a report lost between here and the bus
// is said by whichever host links next. Each declaration it settles is settled after it is said,
// either way. True when the broker took it.
func (q *Queue) tell(ctx context.Context, r Report, kind kindOf, settle ...Declaration) bool {
q.saying.Lock()
defer q.saying.Unlock()
r = q.stamp(r)
keep := kind == applied
if keep {
// What this says supersedes any reconcile's report still waiting to be said.
q.unasked = nil
if q.Unsaid != nil {
if err := q.Unsaid.Keep(r); err != nil {
q.Say("cannot keep this apply's report until it is said: " + err.Error())
}
}
}
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
published := false
if bus != nil {
// **Said even when the link is being let go** (issue 264): not cancelled with it, still
// bounded by the timeout, and the link is not closed until this returns (detach).
published = publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout)
}
if !published && kind == unasked {
q.unasked = &r
}
if published {
// After the report, which is sent first and unchanged; a write the bucket refuses never undoes it (issue 1132).
if q.noteApply(r) {
q.writeHaves(ctx, bus)
}
if keep && q.Unsaid != nil {
if err := q.Unsaid.Said(r.Declared); err != nil {
q.Say("said this apply's report, and cannot forget it: " + err.Error())
}
}
if q.Heard != nil && (kind == applied || kind == unasked) && r.Refused == "" {
q.Heard(r)
}
}
// Settled after the report is published. A node that dies between applying and reporting
// leaves the declaration with the mesh and applies it again on return, which is safe because
// applying is reconciliation — it converges rather than repeating.
for _, d := range settle {
_ = d.Handled()
}
return published
}
// stamp gives a report its node, the next report sequence and the refusals counted. Called holding
// saying.
func (q *Queue) stamp(r Report) Report {
q.load()
q.numbers.sequence++
q.keepNumbers()
r.Node = q.Membership.Node
r.ReportSequence = q.numbers.sequence
r.RefusedOlder = q.numbers.refused
// Every report of a node-engine that orders its reports says it reads a generation (novox/hq issue
// 234): this build does, and the controller sends one only to a machine that said so.
r.ReadsGeneration = true
// And the module of each resource and the build of each module (novox/hq ADR 0429 §4, issue 1132), the same way.
r.ReadsModules = true
return r
}
func (q *Queue) countRefusal() int64 {
q.saying.Lock()
defer q.saying.Unlock()
q.load()
q.numbers.refused++
q.keepNumbers()
return q.numbers.refused
}
// load reads what was counted, once. **Unreadable is said, and the sequence goes on from above
// anything it can have reached** — the time in milliseconds — rather than from one, which would make
// every report this host says next older than what it already said, and refused.
func (q *Queue) load() {
if q.counted {
return
}
q.counted = true
if q.Numbers == nil {
return
}
sequence, refused, err := q.Numbers.Read()
if err != nil {
floor := time.Now().UnixMilli()
q.Say(fmt.Sprintf("cannot read this node's report sequence: %v — going on from %d, above any "+
"it can have reached, and counting refusals again from none", err, floor))
q.numbers = numbered{sequence: floor}
return
}
q.numbers = numbered{sequence: sequence, refused: refused}
}
func (q *Queue) keepNumbers() {
if q.Numbers == nil {
return
}
if err := q.Numbers.Save(q.numbers.sequence, q.numbers.refused); err != nil {
// Said, and not fatal: the report is still worth saying. What is lost is that a successor
// could number one again.
q.Say("cannot keep this node's report sequence: " + err.Error())
}
}
// attach is a link opening. **What the last apply did, if the mesh never heard it**, is said first,
// exactly as it was made; then a reconcile's report that waited; and only then may the worker say
// anything on it — so a report said again can never land after, and read as newer than, a later one.
func (q *Queue) attach(ctx context.Context, bus Bus) {
q.init()
q.saying.Lock()
defer q.saying.Unlock()
reporting := context.WithoutCancel(ctx)
q.load()
said := true
if q.Unsaid != nil {
switch kept, ok, err := q.Unsaid.Pending(); {
case err != nil:
q.Say("cannot read the report this node kept unsaid: " + err.Error())
case ok:
q.Say("saying again what the last apply did: its report never reached the mesh")
if said = publishReport(reporting, bus, q.Membership, kept, q.Say, q.Timeout); said {
if err := q.Unsaid.Said(kept.Declared); err != nil {
q.Say("said the kept report, and cannot forget it: " + err.Error())
}
if q.Heard != nil && kept.Refused == "" {
q.Heard(kept)
}
}
}
}
if said && q.unasked != nil {
if publishReport(reporting, bus, q.Membership, *q.unasked, q.Say, q.Timeout) {
if q.Heard != nil {
q.Heard(*q.unasked)
}
q.unasked = nil
}
}
q.mu.Lock()
q.bus = bus
q.mu.Unlock()
}
// withFiles is ctx carrying the link now open, so an apply can fetch a declaration's files given by digest (novox/hq ADR
// 0486, issue 1695). A queue with no link open carries none.
func (q *Queue) withFiles(ctx context.Context) context.Context {
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return ctx
}
return WithFiles(ctx, bus)
}
// detach is the link closing: it waits for a delivery in hand to be applied and said — an apply that
// stood aside for its successor is reported on this link before it is let go (issue 264) — and then
// nothing more is said on it. A reconcile in hand is not waited for: a link that was dropped or roused
// is opened again at once, and the reconcile's report, if it is news, waits for it.
func (q *Queue) detach(bus Bus) {
q.init()
q.mu.Lock()
for q.delivering {
q.idle.Wait()
}
q.mu.Unlock()
q.saying.Lock()
defer q.saying.Unlock()
q.mu.Lock()
if q.bus == bus {
q.bus = nil
}
q.mu.Unlock()
}
// pick is the newest of what was delivered and the rest, in arrival order, to set aside: by order
// when the declarations claim one, by arrival when they do not (Order.Supersedes).
func pick(batch []Declaration) (Declaration, []Declaration) {
latest := batch[0]
var aside []Declaration
for _, next := range batch[1:] {
if orderOf(next.Body()).Supersedes(orderOf(latest.Body())) {
aside = append(aside, latest)
latest = next
continue
}
aside = append(aside, next)
}
return latest, aside
}
// orderOf is the order a signed declaration claims, or none when it claims none or cannot be read.
// Read from the envelope alone; the signature is verified when the winner is applied, and a forged
// message that lied about its order would only set aside real ones — which are reported as set aside,
// and the next push sends the current one again.
func orderOf(body []byte) Order {
var signed Signed
if err := json.Unmarshal(body, &signed); err != nil {
return Order{}
}
var o Order
if err := json.Unmarshal(signed.Declaration, &o); err != nil || o.Epoch < 0 || o.Sequence < 0 {
return Order{}
}
return o
}
// declaredIn names a signed declaration as the mesh does — the digest of the declaration's bytes, the
// same the apply's report carries — for a report about one that was not applied. Empty if the message
// is not one: a forged or garbled message is refused when its turn comes; here it is only named.
func declaredIn(body []byte) string {
var signed Signed
if err := json.Unmarshal(body, &signed); err != nil || len(signed.Declaration) == 0 {
return ""
}
sum := sha256.Sum256(signed.Declaration)
return hex.EncodeToString(sum[:])
}
// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the
// machine's networking its engine judges).
func (q *Queue) Linked() bool {
q.init()
q.mu.Lock()
defer q.mu.Unlock()
return q.bus != nil
}
// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether
// the bus took it. **Not through the worker**: a statement is a word about the machine as it is, not an
// account of an apply, and it must not wait behind one — a container crash-looping while a long apply
// runs is said while it runs. False with no link open, or one that cannot say it; the caller says it
// again on its next look, and the next report carries it whatever happens.
func (q *Queue) SayHealth(ctx context.Context, h Health) bool {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
sayer, ok := bus.(HealthBus)
if bus == nil || !ok {
return false
}
body, err := json.Marshal(HealthSaid{Node: q.Membership.Node, Health: h})
if err != nil {
return false
}
saying, cancel := context.WithTimeout(ctx, q.timeout())
defer cancel()
if err := sayer.Health(saying, q.Membership.Node, body); err != nil {
q.Say("could not say this machine's health: " + err.Error())
return false
}
// A look that changed a module's health changes its have (issue 1132).
if q.noteHealth(h) {
q.writeHaves(ctx, bus)
}
return true
}
func (q *Queue) timeout() time.Duration {
if q.Timeout <= 0 {
return 10 * time.Second
}
return q.Timeout
}
// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR
// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open
// or nothing answered — said as the check's failure, never as healthy.
func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return health.Answer{}, errors.New("no link to the bus is open")
}
asker, ok := bus.(ToolBus)
if !ok {
return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool")
}
reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}"))
if err != nil {
return health.Answer{}, err
}
return ReadToolAnswer(reply)
}
// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over
// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for.
func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}
asker, ok := bus.(ToolBus)
if !ok {
return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked)
}
return asker.Ask(ctx, CLISubject(q.Membership.Node), body)
}
// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any
// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272).
var ErrNotAsked = errors.New("not asked")
// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call —
// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the
// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the
// SDK cannot check is unhealthy too, so a malformed one never excuses anything.
func ReadToolAnswer(reply []byte) (health.Answer, error) {
var env struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if err := json.Unmarshal(reply, &env); err != nil {
return health.Answer{}, fmt.Errorf("its answer is not one: %w", err)
}
if env.Error != "" {
return health.Unhealthy(env.Error), nil
}
a, err := health.Read(env.Result)
if err != nil {
return health.Unhealthy("it answered something other than healthy or not"), nil
}
return a, nil
}
internal/link/queue_test.go
-588
View File
@@ -1,588 +0,0 @@
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"reflect"
"strings"
"sync"
"testing"
"time"
)
// The node-engine's one apply queue (novox/hq to-be 45 §6): a delivery and the reconcile are two
// reasons to enqueue the same act, the newest declaration is applied once, and the reports leave in
// the order they are made. These are the races of issues 257, 261, 264 and 267 replayed against it.
func aQueue(m Membership, apply Applier, kept Unsaid) *Queue {
return &Queue{Membership: m, Apply: apply, Unsaid: kept, Timeout: time.Second}
}
// runWorker runs the queue's worker until ctx ends; the channel closes when it has.
func runWorker(ctx context.Context, q *Queue) <-chan struct{} {
done := make(chan struct{})
go func() {
defer close(done)
q.Run(ctx)
}()
return done
}
// ordered is a signed declaration claiming an order, with a resource id so two are distinct bytes.
func ordered(t *testing.T, key ed25519.PrivateKey, id string, epoch, sequence int64) *said {
t.Helper()
inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{map[string]any{"id": id}},
"epoch": epoch, "sequence": sequence})
if err != nil {
t.Fatal(err)
}
return &said{body: signedBy(t, key, inner)}
}
// applies records what the worker applied, by the id of the declaration's first resource, and
// reports it as applied — with the order the declaration carried, as the host's own apply does.
type applies struct {
mu sync.Mutex
seen []string
}
func (a *applies) apply(_ context.Context, raw, _ []byte) Report {
var d struct {
Resources []struct {
ID string `json:"id"`
} `json:"resources"`
Order
}
_ = json.Unmarshal(raw, &d)
id := ""
if len(d.Resources) > 0 {
id = d.Resources[0].ID
}
a.mu.Lock()
a.seen = append(a.seen, id)
a.mu.Unlock()
return Report{Declared: digest(raw), Order: d.Order, Applied: []string{id}}
}
func (a *applies) all() []string {
a.mu.Lock()
defer a.mu.Unlock()
return append([]string(nil), a.seen...)
}
func digest(raw []byte) string {
b, _ := json.Marshal(Signed{Declaration: raw})
return declaredIn(b)
}
// eventually waits for a condition the worker reaches on its own time.
func eventually(t *testing.T, what string, ok func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !ok() {
if time.Now().After(deadline) {
t.Fatal(what)
}
time.Sleep(5 * time.Millisecond)
}
}
// accounts is the reports the mesh heard that are an apply's account, not a set-aside's.
func accounts(reports []Report) []Report {
var out []Report
for _, r := range reports {
if r.Superseded == "" {
out = append(out, r)
}
}
return out
}
// **R2: a reconcile due during a delivery** (issues 257, 261, 267). The reconcile is asked for while a
// delivery waits: the worker applies the delivery once — applying the newest thing the mesh said is
// what reconciling is — and makes one report, naming the newest sequence. The reconcile never applies
// what was kept before.
func TestAReconcileDueDuringADeliveryIsThatDeliverysApply(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
reconciled := 0
q := aQueue(m, a.apply, &keptInMemory{})
q.Reconcile = func(context.Context) (Report, bool) {
reconciled++
return Report{Declared: "the one kept before", Applied: []string{"old"}}, true
}
q.News = func(Report) bool { return true }
q.attach(context.Background(), l)
q.Deliver(ordered(t, key, "new", 7, 12))
q.ReconcileDue()
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 })
stop()
<-worker
if got := a.all(); !reflect.DeepEqual(got, []string{"new"}) {
t.Fatalf("applied %v; one apply of the newest was wanted", got)
}
if reconciled != 0 {
t.Fatalf("the reconcile applied what was kept %d time(s) beside the delivery", reconciled)
}
r := l.said()[0]
if r.Sequence != 12 || r.Epoch != 7 || r.ReportSequence != 1 || r.Node != m.Node {
t.Fatalf("the report does not name the order it applied: %+v", r)
}
}
// **The other order** (issue 267): the reconcile is running when a delivery arrives. Its report, about
// the declaration kept then, leaves first; the delivery's leaves after it, with a higher report
// sequence. The mesh's latest word is the newest declaration, whichever arrived first.
func TestAReconcileRunningWhenADeliveryArrivesIsSaidBeforeIt(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
q := aQueue(m, a.apply, &keptInMemory{})
inReconcile, release := make(chan struct{}), make(chan struct{})
q.Reconcile = func(context.Context) (Report, bool) {
close(inReconcile)
<-release
return Report{Declared: "d1", Order: Order{Epoch: 7, Sequence: 11}, Applied: []string{"a"},
Outward: []string{"eth0"}}, true
}
q.News = func(Report) bool { return true }
q.attach(context.Background(), l)
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
q.ReconcileDue()
<-inReconcile
q.Deliver(ordered(t, key, "b", 7, 12))
close(release)
eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 2 })
stop()
<-worker
reports := l.said()
if reports[0].Declared != "d1" || reports[1].Sequence != 12 {
t.Fatalf("the reports left out of the order they were made: %+v", reports)
}
if reports[0].ReportSequence >= reports[1].ReportSequence {
t.Fatalf("the later report carries no higher report sequence: %d then %d",
reports[0].ReportSequence, reports[1].ReportSequence)
}
}
// **Two deliveries in quick succession**: only the newest is applied, and there is one account of an
// apply — the first is reported as set aside, naming the one that took its place.
func TestTwoDeliveriesInQuickSuccessionApplyOnlyTheNewest(t *testing.T) {
m, key := aMember(t)
first, second := ordered(t, key, "first", 7, 12), ordered(t, key, "second", 7, 13)
l := newQuietLink(first, second)
a := &applies{}
q := aQueue(m, a.apply, &keptInMemory{})
ctx, stop := context.WithCancel(context.Background())
defer stop()
worker := runWorker(ctx, q)
go func() { _ = serve(ctx, l, m, q, nil, time.Second) }()
eventually(t, "nothing was reported", func() bool { return len(accounts(l.said())) == 1 })
stop()
<-worker
if got := a.all(); !reflect.DeepEqual(got, []string{"second"}) {
t.Fatalf("applied %v; only the newest was wanted", got)
}
reports := l.said()
if len(reports) != 2 || reports[0].Superseded != declaredIn(second.body) ||
reports[0].Declared != declaredIn(first.body) || reports[0].Sequence != 12 {
t.Fatalf("the first was not reported as set aside for the second: %+v", reports)
}
if !first.wasHandled() || !second.wasHandled() {
t.Fatal("a declaration was left unsettled")
}
}
// Deliveries that arrive while the worker is applying are coalesced: when it is free it takes the
// newest held at that moment — by order, not by arrival — and applies it once.
func TestDeliveriesWhileTheWorkerIsBusyAreOneApplyOfTheNewest(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
inApply, release := make(chan struct{}, 1), make(chan struct{})
q := aQueue(m, func(ctx context.Context, raw, sig []byte) Report {
r := a.apply(ctx, raw, sig)
if r.Sequence == 1 {
inApply <- struct{}{}
<-release
}
return r
}, &keptInMemory{})
q.attach(context.Background(), l)
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
q.Deliver(ordered(t, key, "one", 7, 1))
<-inApply
q.Deliver(ordered(t, key, "three", 7, 3))
q.Deliver(ordered(t, key, "two", 7, 2)) // arrived last, composed earlier
q.ReconcileDue() // and the five-minute timer fired too
close(release)
eventually(t, "the newest was never reported", func() bool { return len(accounts(l.said())) == 2 })
stop()
<-worker
if got := a.all(); !reflect.DeepEqual(got, []string{"one", "three"}) {
t.Fatalf("applied %v; the one in hand and then only the newest were wanted", got)
}
last := accounts(l.said())[1]
if last.Sequence != 3 {
t.Fatalf("the last account names sequence %d, not the newest", last.Sequence)
}
}
// **An older epoch is refused, counted and said** (rule 2): the refusal's report names the refused
// declaration and what this node holds, carries the count, and does not replace the kept account of
// the last apply — which is what the mesh is waiting for from this node.
func TestADeclarationFromAnOlderEpochIsRefusedCountedAndSaid(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
kept := &keptInMemory{}
lastApply := Report{Declared: "d-applied", Order: Order{Epoch: 57, Sequence: 3}, Applied: []string{"a"}}
_ = kept.Keep(lastApply)
stale := ordered(t, key, "stale", 41, 12)
q := aQueue(m, func(_ context.Context, raw, _ []byte) Report {
// What the host's apply answers for a declaration older than the one it kept.
return Report{Declared: digest(raw), Order: Order{Epoch: 41, Sequence: 12},
OlderThan: &Order{Epoch: 57, Sequence: 3}, Refused: "older than what this node applied"}
}, kept)
var lines saidSoFar
q.Say = lines.say
l.refusing = true // nothing is said on linking: the kept account stays kept
q.attach(context.Background(), l)
l.mu.Lock()
l.refusing = false
l.mu.Unlock()
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
q.Deliver(stale)
eventually(t, "the refusal was never reported", func() bool { return len(l.said()) == 1 })
stop()
<-worker
r := l.said()[0]
if r.OlderThan == nil || *r.OlderThan != (Order{Epoch: 57, Sequence: 3}) || r.Epoch != 41 ||
r.Declared != declaredIn(stale.body) || r.RefusedOlder != 1 {
t.Fatalf("the refusal does not name what was refused, what is held, and the count: %+v", r)
}
if !strings.Contains(lines.all(), "1 refused as older so far") {
t.Fatalf("the refusal was not said in the log:\n%s", lines.all())
}
if still, ok, _ := kept.Pending(); !ok || still.Declared != "d-applied" {
t.Fatalf("the refusal replaced the kept account of the last apply: %+v", still)
}
if !stale.wasHandled() {
t.Fatal("the refused declaration was not settled: the mesh would deliver it again")
}
}
// **Restart with an unsaid report**: the next host says it first, exactly as it was made, and numbers
// what it says next above it — the report sequence goes on across the restart rather than from one.
func TestAfterARestartTheUnsaidReportIsSaidFirstAndTheNumbersGoOn(t *testing.T) {
m, key := aMember(t)
kept := &keptInMemory{}
numbers := &numbersInMemory{}
// The first host applies; its report never reaches the mesh, and it is gone.
first := newQuietLink()
first.refusing = true
q1 := aQueue(m, (&applies{}).apply, kept)
q1.Numbers = numbers
q1.attach(context.Background(), first)
ctx1, stop1 := context.WithCancel(context.Background())
w1 := runWorker(ctx1, q1)
q1.Deliver(ordered(t, key, "a", 7, 12))
eventually(t, "the first host never kept its report", func() bool { _, ok, _ := kept.Pending(); return ok })
stop1()
<-w1
lost, _, _ := kept.Pending()
// The next host links, and is then delivered something new.
next := newQuietLink()
q2 := aQueue(m, (&applies{}).apply, kept)
q2.Numbers = numbers
q2.attach(context.Background(), next)
ctx2, stop2 := context.WithCancel(context.Background())
w2 := runWorker(ctx2, q2)
q2.Deliver(ordered(t, key, "b", 7, 13))
eventually(t, "the next host's apply was never reported", func() bool { return len(next.said()) == 2 })
stop2()
<-w2
reports := next.said()
if !reflect.DeepEqual(reports[0], lost) {
t.Fatalf("the unsaid report was not said first, as it was made: %+v", reports[0])
}
if reports[1].Sequence != 13 || reports[1].ReportSequence <= lost.ReportSequence {
t.Fatalf("the next host numbered from one again: %d after %d",
reports[1].ReportSequence, lost.ReportSequence)
}
}
// A report sequence that cannot be read is said, and numbering goes on above anything it can have
// reached — never from one, which would make every report older than those already said.
func TestUnreadableNumbersAreSaidAndNumberingGoesOnAboveThem(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
q := aQueue(m, nil, nil)
q.Numbers = &numbersInMemory{broken: errors.New("unreadable")}
var lines saidSoFar
q.Say = lines.say
q.News = func(Report) bool { return true }
q.Reconcile = func(context.Context) (Report, bool) { return Report{Declared: "d"}, true }
before := time.Now().UnixMilli()
q.attach(context.Background(), l)
q.ReconcileDue()
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
eventually(t, "nothing was reported", func() bool { return len(l.said()) == 1 })
stop()
<-worker
if got := l.said()[0].ReportSequence; got < before {
t.Fatalf("numbering went on from %d, not above anything it can have reached", got)
}
if !strings.Contains(lines.all(), "cannot read this node's report sequence") {
t.Fatalf("the unreadable sequence was not said:\n%s", lines.all())
}
}
// A reconcile's report made with no link waits for the next one; an apply made before then replaces
// it, because the apply's account is fresher.
func TestAReconcileReportMadeOfflineWaitsAndIsReplacedByAnApply(t *testing.T) {
m, key := aMember(t)
q := aQueue(m, (&applies{}).apply, &keptInMemory{})
q.News = func(Report) bool { return true }
q.Reconcile = func(context.Context) (Report, bool) {
return Report{Declared: "d1", Outward: []string{"eth0"}}, true
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
runWorker(ctx, q)
q.ReconcileDue()
eventually(t, "the reconcile's report was not held", func() bool {
q.saying.Lock()
defer q.saying.Unlock()
return q.unasked != nil
})
l := newQuietLink()
q.attach(context.Background(), l)
if reports := l.said(); len(reports) != 1 || reports[0].Declared != "d1" {
t.Fatalf("the reconcile's report was not said when the link opened: %+v", reports)
}
// Offline again: a reconcile's report, then an apply of a delivery still in hand.
q.detach(l)
q.ReconcileDue()
eventually(t, "the second reconcile's report was not held", func() bool {
q.saying.Lock()
defer q.saying.Unlock()
return q.unasked != nil
})
q.Deliver(ordered(t, key, "x", 7, 2))
eventually(t, "the apply did not replace the reconcile's report", func() bool {
q.saying.Lock()
defer q.saying.Unlock()
return q.unasked == nil
})
}
// The contract with the controller, on the wire: the keys a declaration's order is read from, and the
// keys a report carries back. A rename here must break this test before it breaks a machine.
func TestTheOrderOnTheWire(t *testing.T) {
body, err := json.Marshal(Report{Node: "n", Declared: "d", Order: Order{Epoch: 57, Sequence: 3},
ReportSequence: 9, OlderThan: &Order{Epoch: 58, Sequence: 1}, RefusedOlder: 2})
if err != nil {
t.Fatal(err)
}
var wire map[string]any
_ = json.Unmarshal(body, &wire)
want := map[string]any{"node": "n", "declared": "d", "epoch": 57.0, "sequence": 3.0,
"report_sequence": 9.0, "older_than": map[string]any{"epoch": 58.0, "sequence": 1.0},
"refused_older": 2.0}
if !reflect.DeepEqual(wire, want) {
t.Fatalf("a report's order on the wire is\n%s\nnot the contract", body)
}
// An older host's report, and an older controller's declaration, claim no order.
if body, _ := json.Marshal(Report{Node: "n", Declared: "d"}); strings.Contains(string(body), "sequence") ||
strings.Contains(string(body), "epoch") {
t.Fatalf("a report about a declaration with no order claims one: %s", body)
}
_, key := aMember(t)
if got := orderOf(signedBy(t, key, []byte(`{"declaration":1,"epoch":57,"sequence":3}`))); got != (Order{Epoch: 57, Sequence: 3}) {
t.Fatalf("a declaration's order was read as %+v", got)
}
}
func TestOlder(t *testing.T) {
for _, c := range []struct {
in, held Order
older bool
}{
{Order{41, 12}, Order{57, 3}, true}, // an older lease holder, whatever its sequence
{Order{57, 2}, Order{57, 3}, true}, // same holder, lower sequence
{Order{57, 3}, Order{57, 3}, false}, // the same declaration again: reconciling
{Order{58, 1}, Order{57, 3}, false}, // a new lease holder
{Order{0, 1}, Order{57, 3}, false}, // an older controller, or one rolled back: today's behaviour
{Order{41, 1}, Order{0, 9}, false}, // nothing held claims an epoch
{Order{57, 0}, Order{57, 3}, false}, // no sequence to compare
{Order{0, 0}, Order{0, 0}, false}, // neither claims anything
{Order{-1, 0}, Order{57, 3}, false}, // never a claim
{Order{57, 4}, Order{57, 3}, false}, // newer
{Order{56, 99}, Order{57, 1}, true}, // a higher sequence is no excuse for an older epoch
{Order{57, 1}, Order{56, 99}, false}, // nor a lower one a reason to refuse a newer epoch
{Order{57, 3}, Order{57, 0}, false}, // held with no sequence
{Order{100, 0}, Order{57, 0}, false}, // newer epoch, no sequences
{Order{10, 0}, Order{57, 0}, true}, // older epoch, no sequences
{Order{57, 2}, Order{0, 0}, false}, // nothing held at all
{Order{0, 2}, Order{0, 3}, false}, // sequence alone does not refuse on the link (no epoch)
{Order{57, 2}, Order{57, -1}, false}, // a held sequence below zero is not one
{Order{57, -1}, Order{57, 2}, false}, // nor an arriving one
{Order{-5, -5}, Order{-1, -1}, false}, // nothing below zero is an order
} {
if got := c.in.Older(c.held); got != c.older {
t.Errorf("%+v older than %+v = %v, want %v", c.in, c.held, got, c.older)
}
}
}
func TestSupersedes(t *testing.T) {
for _, c := range []struct {
next, before Order
want bool
}{
{Order{58, 1}, Order{57, 9}, true}, // a new lease holder
{Order{57, 9}, Order{58, 1}, false}, // the stale one arriving late
{Order{57, 4}, Order{57, 3}, true},
{Order{57, 2}, Order{57, 3}, false}, // arrived last, composed earlier
{Order{0, 4}, Order{0, 3}, true},
{Order{0, 2}, Order{0, 3}, false},
{Order{}, Order{0, 3}, true}, // no order: by arrival
{Order{0, 3}, Order{}, true},
} {
if got := c.next.Supersedes(c.before); got != c.want {
t.Errorf("%+v supersedes %+v = %v, want %v", c.next, c.before, got, c.want)
}
}
}
// numbersInMemory is Numbers without a disk, surviving a "restart" by being shared.
type numbersInMemory struct {
mu sync.Mutex
sequence, refused int64
broken error
}
func (n *numbersInMemory) Read() (int64, int64, error) {
n.mu.Lock()
defer n.mu.Unlock()
return n.sequence, n.refused, n.broken
}
func (n *numbersInMemory) Save(sequence, refused int64) error {
n.mu.Lock()
defer n.mu.Unlock()
n.sequence, n.refused = sequence, refused
return nil
}
// A link dropped or roused while a reconcile is in hand is let go at once and opened again: the
// reconcile holds no link, and its report, if it is news, waits for the next one. Only a delivery in
// hand keeps its link until its report is said.
func TestALinkIsNotHeldForAReconcileInHand(t *testing.T) {
m, _ := aMember(t)
inReconcile, release := make(chan struct{}), make(chan struct{})
q := aQueue(m, nil, nil)
q.News = func(Report) bool { return true }
q.Reconcile = func(context.Context) (Report, bool) {
close(inReconcile)
<-release
return Report{Declared: "d1", Outward: []string{"eth0"}}, true
}
l := newQuietLink()
q.attach(context.Background(), l)
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
q.ReconcileDue()
<-inReconcile
detached := make(chan struct{})
go func() { q.detach(l); close(detached) }()
select {
case <-detached:
case <-time.After(2 * time.Second):
t.Fatal("the link was held while a reconcile ran")
}
close(release)
eventually(t, "the reconcile's report was not held for the next link", func() bool {
q.saying.Lock()
defer q.saying.Unlock()
return q.unasked != nil
})
stop()
<-worker
if len(l.said()) != 0 {
t.Fatalf("a report was said on a link let go: %+v", l.said())
}
}
// **Every report says this node-engine reads a declaration's generation** (novox/hq issue 234): the
// controller sends one only to a machine that said so, because an older node-engine refuses the key.
func TestEveryReportSaysItReadsAGeneration(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
q := aQueue(m, a.apply, &keptInMemory{})
q.attach(context.Background(), l)
q.Deliver(ordered(t, key, "new", 7, 12))
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 })
stop()
<-worker
raw, err := json.Marshal(l.said()[0])
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"reads_generation":true`) {
t.Fatalf("the report does not say it reads a generation: %s", raw)
}
}
// 1132/AC2: every report says this node-engine reads a resource's module and a declaration's builds, beside the
// generation: the controller sends them only to a machine that said so, because an older node-engine refuses the keys.
func TestEveryReportSaysItReadsModules(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
q := aQueue(m, a.apply, &keptInMemory{})
q.attach(context.Background(), l)
q.Deliver(ordered(t, key, "new", 7, 12))
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 })
stop()
<-worker
raw, err := json.Marshal(l.said()[0])
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"reads_modules":true`) || !strings.Contains(string(raw), `"reads_generation":true`) {
t.Fatalf("the report does not say it reads modules beside the generation: %s", raw)
}
}
internal/link/replay331_test.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
// The NATS link is attached with no connection under it, so no bus is needed: the question must reach the
// link and fail there, not stop at the queue as if no link were open.
func TestReplay331(t *testing.T) {
q := &Queue{Membership: Membership{Node: "laptop"}}
q := &Session{Membership: Membership{Node: "laptop"}}
q.attach(t.Context(), &natsLink{node: "laptop"})
_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
if err == nil {
internal/link/run.go
+10 -165
View File
@@ -6,18 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
)
// ErrForged is what a node returns for a declaration whose signature is not the mesh's.
//
// Its own error, and it must never be confused with a malformed message. novox/hq ADR 0004
// requires a host to tell *this is not from the mesh I joined* apart from *this is malformed*:
// the first means somebody is trying, the second means something is broken.
var ErrForged = errors.New("this declaration was not signed by the mesh this node joined")
// Membership is what a node needs to reach its mesh again, held by the caller.
type Membership struct {
Node string
@@ -30,13 +21,6 @@ type Membership struct {
Transport string
}
// Applier is what the host does with a declaration that has been proved to come from the mesh.
//
// It receives the signature as well as the declaration, so the host can keep both: what it was
// told is kept signed and verified again when it is read back, which means the file on disk is
// trusted for the same reason the message was rather than for being local.
type Applier func(ctx context.Context, declaration, signature []byte) Report
// Announce is how the link says what is happening, so a node running unattended leaves an
// account of it. Nil is allowed and means say nothing.
type Announce func(string)
@@ -64,35 +48,14 @@ type Announce func(string)
// Nil is allowed and means nothing ever rouses it, which is every machine that does not suspend.
type Roused <-chan struct{}
func Hold(ctx context.Context, m Membership, queue *Queue, say Announce, timeout time.Duration) error {
func Hold(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration) error {
return HoldRoused(ctx, m, queue, say, timeout, nil)
}
// Unsaid keeps the report of the last apply until the mesh has taken it, so a report lost between
// applying and publishing — the host standing aside for a successor, a crash, a power cut — is said
// again the next time this node is linked (novox/hq issue 264). Without it the declaration has been
// settled or applied, the machine is what the mesh said, and the mesh waits for a report that the
// node believes is gone.
//
// **Exactly the report the apply made, never a new one.** Re-applying the kept declaration would
// describe the machine as it is now; what the mesh waits for is what was done with what it sent.
// Nil is allowed and keeps nothing.
type Unsaid interface {
// Keep holds the report of an apply until it is said. A report naming no declaration — refused
// before it was read — replaces nothing worth saying, and clears what was kept.
Keep(Report) error
// Said forgets the kept report once the broker has taken the report about the same declaration.
Said(declared string) error
// Pending is the report kept and not yet said, if any.
Pending() (Report, bool, error)
}
// HoldRoused is Hold, told when the machine has reason to think its link is stale.
//
// **What arrives is enqueued, never applied here** (novox/hq to-be 45 §6). The queue's one worker
// applies, outlives every link, and says its reports on whichever link is open; the caller runs it
// (Queue.Run) for as long as this holds.
func HoldRoused(ctx context.Context, m Membership, queue *Queue, say Announce,
// The session outlives every link and says the node's facts on whichever is open (novox/hq ADR 0539 §9).
func HoldRoused(ctx context.Context, m Membership, queue *Session, say Announce,
timeout time.Duration, roused Roused) error {
return holdWith(ctx, func(ctx context.Context) error {
@@ -185,11 +148,11 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err
}
}
// Run holds the link open once, applying what arrives and reporting what happened.
// Run holds the link open once, hearing what the controller asks and saying the node's facts.
//
// Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason;
// Hold is what decides whether to open it again.
func Run(ctx context.Context, m Membership, queue *Queue, say Announce, timeout time.Duration) error {
func Run(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration) error {
link, err := Open(ctx, m, timeout)
if err != nil {
return err
@@ -200,7 +163,7 @@ func Run(ctx context.Context, m Membership, queue *Queue, say Announce, timeout
// serve is Run on a link already open: separated so what the node says, and when, can be tested
// without a broker.
func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Announce,
func serve(ctx context.Context, link Link, m Membership, queue *Session, say Announce,
timeout time.Duration) error {
if say == nil {
say = func(string) {}
@@ -209,18 +172,13 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou
// Said, because it is the event anybody watching actually wants. Without it a node logs every
// failure and nothing on success, so a log full of "trying again" and then silence reads as
// still broken when it means the opposite.
say("in the mesh, hearing what this node should be")
say("in the mesh, hearing what the controller asks of this node")
// **What the last apply did, if the mesh never heard it**, is said before anything newly
// delivered can be: the queue says it as the link is handed to it (novox/hq issue 264). And the
// link is not let go while the queue's worker is mid-act: an apply that ends the link — the host
// standing aside for a successor it delivered (novox/hq ADR 0141) — is still reported on it,
// before the deferred close.
// The session says this node's facts as the link is handed to it, and is not let go while it is mid-act.
queue.attach(ctx, link)
defer queue.detach(link)
declarations := link.Declarations()
// And the mesh asking what this node last applied (to-be 45 §6), on a link that hears the question.
// And the mesh asking this node to say its facts again, on a link that hears the question.
var asks <-chan struct{}
if a, ok := link.(Asked); ok {
asks = a.AskedToReport()
@@ -248,7 +206,6 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou
}
for {
// Asked to stop — by standing aside, say — nothing further is taken in.
if ctx.Err() != nil {
return nil
}
@@ -258,90 +215,11 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou
case reason := <-link.Lost():
return reason
case <-asks:
queue.ReportAsked()
case declaration, ok := <-declarations:
if !ok {
// The link's own reason, when it has managed to say one: "stopped delivering" on
// its own says nothing about why, and why is the whole of what an operator wants.
select {
case reason := <-link.Lost():
return reason
default:
return errors.New("the mesh stopped sending this node declarations")
}
}
// Whatever else is already waiting goes with it. **Enqueued, not applied**: the queue
// applies the newest of everything delivered and not yet taken, and reports the rest as
// set aside (novox/hq to-be 45 §6).
queue.Deliver(gather(declarations, declaration, drainWindow)...)
queue.FactsAsked(ctx)
}
}
}
// short is a declaration's digest as a person reads it in a log line.
func short(digest string) string {
if len(digest) > 12 {
return digest[:12]
}
return digest
}
// drainDepth is how many declarations the link holds unread; drainWindow is how long it waits for
// another to follow the one it has. Both small: a push is rare and a backlog is the exception this
// exists for, not the shape of ordinary traffic.
const (
drainDepth = 16
drainWindow = 750 * time.Millisecond
)
// gather takes what is already waiting behind `first`, in the order it arrived. It waits `window`
// for a straggler after each arrival and no longer: a declaration in flight from the mesh arrives
// within that; one that does not is the next push. Which of them is applied is the queue's to decide
// (pick), across everything delivered and not yet taken.
//
// **Its job narrows once declarations are state rather than messages, and does not disappear.**
// On the bus a declaration is last-per-subject (novox/hq design 29 §4), so a node that was away
// receives exactly the current one instead of a queue of superseded ones — the catch-up half is the
// stream's. And a declaration's order decides which is newest, where this window only infers it from
// arrival time, which is the wire-level answer to novox/hq issue 107.
//
// What remains is the live case: three pushes in quick succession to a *connected*, idle node are
// three deliveries, whatever the stream later retains, and gathering them is what makes them one
// apply rather than two. So this is narrowed, not deleted — and saying which half goes is worth more
// than a note that it "can probably be removed", which is how a load-bearing window gets deleted by
// somebody in a hurry.
func gather(arriving <-chan Declaration, first Declaration, window time.Duration) []Declaration {
batch := []Declaration{first}
for {
select {
case next, ok := <-arriving:
if !ok {
return batch
}
batch = append(batch, next)
case <-time.After(window):
return batch
}
}
}
// handleBody is the whole of deciding whether to trust a message, separated from the broker so it
// can be tested as the security check it is rather than as message plumbing.
func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report {
var signed Signed
if err := json.Unmarshal(body, &signed); err != nil {
return Report{Node: m.Node, Refused: "this message is not a declaration: " + err.Error()}
}
// Before anything is read out of it, let alone applied. The host applies whatever the link
// delivers, so this check is the difference between the mesh changing this machine and
// anybody changing it.
if !ed25519.Verify(m.Signer, signed.Declaration, signed.Signature) {
return Report{Node: m.Node, Refused: ErrForged.Error()}
}
return apply(ctx, signed.Declaration, signed.Signature)
}
// publishReport tells the mesh what this node did, and says whether the broker took it.
// Publish sends one report on this node's own connection and returns: the one-shot path for a
// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same
@@ -379,36 +257,3 @@ func publishReport(ctx context.Context, bus Bus, m Membership, report Report,
}
return true
}
// heldNote is what this apply did NOT do, for the line that says what it did.
//
// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125).
// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but
// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources,
// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took
// opening state.json by hand; not reading it took every public name on the machine down, because the
// operator had four green surfaces and a discrepancy nobody could interpret.
//
// So the line that reports the apply carries it. Grouped by module and ordered by name, because the
// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing
// they can act on — `take` is the verb, and it takes a module.
func heldNote(held []Held) string {
if len(held) == 0 {
return ""
}
byModule := map[string]int{}
for _, h := range held {
byModule[h.Module]++
}
names := make([]string, 0, len(byModule))
for name := range byModule {
names = append(names, name)
}
sort.Strings(names)
parts := make([]string, 0, len(names))
for _, name := range names {
parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name]))
}
return fmt.Sprintf(", %d held until their module is taken (%s)",
len(held), strings.Join(parts, ", "))
}
internal/link/session.go
+257
View File
@@ -0,0 +1,257 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program
// that wrote them, beside its epoch and sequence.
const Writer = "node-engine"
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks
// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts.
//
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an
// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles
// a whole declaration.
type Session struct {
// Membership is the node and the mesh's key a controller's ask is verified with.
Membership Membership
// OnLink is run, off the link's own loop, each time a link opens and when the controller asks the node to say its
// facts again. It says them with SayFacts. Nil says nothing.
OnLink func(ctx context.Context)
// Heard is told every facts event the mesh has taken. Nil is allowed.
Heard func(Report)
// HaveWritten is told each time a have key is written, or found already saying what this node-engine would write:
// the first proves a node-engine on trial (novox/hq ADR 0539 §9). Nil is allowed.
HaveWritten func()
// HandOver judges and records a directory used as found handed to the mesh by the operator, asked by
// the controller over the link (novox/hq issue 356). Nil answers every such ask with a refusal.
HandOver HandOverFunc
// SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the
// controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal.
SetuidSearch SetuidSearchFunc
// JudgeDirectories judges this node's declared directories read-only, asked by the controller over the
// link (novox/hq issue 358). Nil answers every such ask with a refusal.
JudgeDirectories JudgeDirectoriesFunc
// AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved
// warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal.
AccessAct AccessActFunc
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
MoveSecret MoveSecretFunc
Say Announce
Timeout time.Duration
once sync.Once
mu sync.Mutex
bus Bus // the link open now; nil while there is none
// saying is held while a facts event is made and said, so they leave in the order they are made.
saying sync.Mutex
// epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one
// without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2).
epoch int64
sequence int64
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
haves haveState
}
func (q *Session) init() {
q.once.Do(func() {
if q.Say == nil {
q.Say = func(string) {}
}
q.epoch = time.Now().UnixMilli()
})
}
// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them.
// A facts event made with no link open is not kept: the next link says the facts as they are then.
func (q *Session) SayFacts(ctx context.Context, r Report) bool {
q.init()
q.saying.Lock()
defer q.saying.Unlock()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return false
}
q.sequence++
r.Node = q.Membership.Node
r.Writer = Writer
r.Order = Order{Epoch: q.epoch, Sequence: q.sequence}
if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) {
return false
}
// After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132).
if q.noteApply(r) {
q.writeHaves(ctx, bus)
}
if q.Heard != nil && r.Refused == "" {
q.Heard(r)
}
return true
}
// FactsAsked is the mesh asking this node to say its facts again: run on the link open now.
func (q *Session) FactsAsked(ctx context.Context) {
q.init()
if q.OnLink != nil {
go q.OnLink(ctx)
}
}
// attach is a link opening: it is the link facts are said on from now.
func (q *Session) attach(ctx context.Context, bus Bus) {
q.init()
q.mu.Lock()
q.bus = bus
q.mu.Unlock()
q.FactsAsked(ctx)
}
// WithLink is ctx carrying the link now open, so an apply can fetch a composition's files given by digest (novox/hq ADR
// 0486, issue 1695). A session with no link open carries none.
func (q *Session) WithLink(ctx context.Context) context.Context {
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return ctx
}
return WithFiles(ctx, bus)
}
// detach is the link closing: nothing more is said on it.
func (q *Session) detach(bus Bus) {
q.init()
q.saying.Lock()
defer q.saying.Unlock()
q.mu.Lock()
if q.bus == bus {
q.bus = nil
}
q.mu.Unlock()
}
// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the
// machine's networking its engine judges).
func (q *Session) Linked() bool {
q.init()
q.mu.Lock()
defer q.mu.Unlock()
return q.bus != nil
}
// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether
// the bus took it. A statement is a word about the machine as it is, and it must not wait behind an apply: a
// container crash-looping while a long apply runs is said while it runs. False with no link open, or one that cannot
// say it; the caller says it again on its next look, and the next facts event carries it whatever happens.
func (q *Session) SayHealth(ctx context.Context, h Health) bool {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
sayer, ok := bus.(HealthBus)
if bus == nil || !ok {
return false
}
body, err := json.Marshal(HealthSaid{Node: q.Membership.Node, Health: h})
if err != nil {
return false
}
saying, cancel := context.WithTimeout(ctx, q.timeout())
defer cancel()
if err := sayer.Health(saying, q.Membership.Node, body); err != nil {
q.Say("could not say this machine's health: " + err.Error())
return false
}
// A look that changed a module's health changes its have (issue 1132).
if q.noteHealth(h) {
q.writeHaves(ctx, bus)
}
return true
}
func (q *Session) timeout() time.Duration {
if q.Timeout <= 0 {
return 10 * time.Second
}
return q.Timeout
}
// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR
// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open
// or nothing answered — said as the check's failure, never as healthy.
func (q *Session) AskTool(ctx context.Context, module, tool string) (health.Answer, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return health.Answer{}, errors.New("no link to the bus is open")
}
asker, ok := bus.(ToolBus)
if !ok {
return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool")
}
reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}"))
if err != nil {
return health.Answer{}, err
}
return ReadToolAnswer(reply)
}
// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over
// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for.
func (q *Session) AskController(ctx context.Context, body []byte) ([]byte, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}
asker, ok := bus.(ToolBus)
if !ok {
return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked)
}
return asker.Ask(ctx, CLISubject(q.Membership.Node), body)
}
// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any
// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272).
var ErrNotAsked = errors.New("not asked")
// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call —
// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the
// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the
// SDK cannot check is unhealthy too, so a malformed one never excuses anything.
func ReadToolAnswer(reply []byte) (health.Answer, error) {
var env struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if err := json.Unmarshal(reply, &env); err != nil {
return health.Answer{}, fmt.Errorf("its answer is not one: %w", err)
}
if env.Error != "" {
return health.Unhealthy(env.Error), nil
}
a, err := health.Read(env.Result)
if err != nil {
return health.Unhealthy("it answered something other than healthy or not"), nil
}
return a, nil
}
internal/link/session_test.go
+164
View File
@@ -0,0 +1,164 @@
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"os"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled,
// as the bus's is ("reporting: context canceled"), or while refusing.
type quietLink struct {
mu sync.Mutex
reports []Report
refusing bool
lost chan error
}
func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error {
if err := ctx.Err(); err != nil {
return err
}
l.mu.Lock()
defer l.mu.Unlock()
if l.refusing {
return errors.New("refused")
}
var r Report
if err := json.Unmarshal(body, &r); err != nil {
return err
}
l.reports = append(l.reports, r)
return nil
}
func (l *quietLink) Lost() <-chan error { return l.lost }
func (l *quietLink) Close() {}
func (l *quietLink) said() []Report {
l.mu.Lock()
defer l.mu.Unlock()
return append([]Report(nil), l.reports...)
}
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
t.Helper()
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
return Membership{Node: "n1", Signer: public}, private
}
// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539
// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts.
func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
s := &Session{Membership: m, Timeout: time.Second}
s.attach(context.Background(), l)
if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) {
t.Fatal("the facts were not taken")
}
got := l.said()
if len(got) != 2 {
t.Fatalf("%d facts events were said", len(got))
}
for _, r := range got {
if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 {
t.Fatalf("the facts do not say who wrote them: %+v", r)
}
}
if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 {
t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order)
}
// A later start orders after this one with nothing kept on disk.
time.Sleep(2 * time.Millisecond)
later := &Session{Membership: m, Timeout: time.Second}
later.attach(context.Background(), l)
later.SayFacts(context.Background(), Report{})
if last := l.said()[2]; last.Epoch <= got[1].Epoch {
t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch)
}
}
// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then.
func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
s := &Session{Membership: m, Timeout: time.Second}
if s.SayFacts(context.Background(), Report{}) {
t.Fatal("facts were said with no link")
}
s.attach(context.Background(), l)
s.detach(l)
if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 {
t.Fatal("facts were said on a link that was let go")
}
}
// A link opening says the node's facts, off the link's own loop.
func TestALinkOpeningSaysTheFacts(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
opened := make(chan struct{}, 1)
s := &Session{Membership: m, Timeout: time.Second, OnLink: func(context.Context) { opened <- struct{}{} }}
s.attach(context.Background(), l)
select {
case <-opened:
case <-time.After(2 * time.Second):
t.Fatal("a link opening did not ask for the facts")
}
}
// aBus is a connection to a real server, because some claims here are about one: set MESH_TEST_NATS to a server with
// JetStream, or the test is skipped.
func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
t.Helper()
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
Retention: nats.WorkQueuePolicy}
if _, err := js.StreamInfo(want.Name); err != nil {
if _, err := js.AddStream(want); err != nil {
t.Fatal(err)
}
}
if err := js.PurgeStream(want.Name); err != nil {
t.Fatal(err)
}
return conn, js
}
// eventuallyHeld waits for a condition the bus reaches on its own time.
func eventuallyHeld(t *testing.T, what string, ok func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !ok() {
if time.Now().After(deadline) {
t.Fatal(what)
}
time.Sleep(5 * time.Millisecond)
}
}
internal/link/unsaid_test.go
-217
View File
@@ -1,217 +0,0 @@
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"reflect"
"sync"
"testing"
"time"
)
// A report of an apply reaches the mesh even when the apply ended the link, and one lost anyway is
// said again the next time the node is linked (novox/hq issue 264).
// quietLink is a link with no broker behind it. A report is refused once the context it is
// published on is cancelled, as the bus's is ("reporting: context canceled"), or while refusing.
type quietLink struct {
mu sync.Mutex
reports []Report
refusing bool
delivered chan Declaration
lost chan error
}
func newQuietLink(arriving ...Declaration) *quietLink {
l := &quietLink{delivered: make(chan Declaration, len(arriving)), lost: make(chan error, 1)}
for _, d := range arriving {
l.delivered <- d
}
return l
}
func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error {
if err := ctx.Err(); err != nil {
return err
}
l.mu.Lock()
defer l.mu.Unlock()
if l.refusing {
return errors.New("refused")
}
var r Report
if err := json.Unmarshal(body, &r); err != nil {
return err
}
l.reports = append(l.reports, r)
return nil
}
func (l *quietLink) Declarations() <-chan Declaration { return l.delivered }
func (l *quietLink) Lost() <-chan error { return l.lost }
func (l *quietLink) Close() {}
func (l *quietLink) said() []Report {
l.mu.Lock()
defer l.mu.Unlock()
return append([]Report(nil), l.reports...)
}
// keptInMemory is Unsaid without a disk.
type keptInMemory struct {
mu sync.Mutex
report *Report
}
func (k *keptInMemory) Keep(r Report) error {
k.mu.Lock()
defer k.mu.Unlock()
if r.Declared == "" {
k.report = nil
return nil
}
k.report = &r
return nil
}
func (k *keptInMemory) Said(declared string) error {
k.mu.Lock()
defer k.mu.Unlock()
if k.report != nil && k.report.Declared == declared {
k.report = nil
}
return nil
}
func (k *keptInMemory) Pending() (Report, bool, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.report == nil {
return Report{}, false, nil
}
return *k.report, true, nil
}
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
t.Helper()
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
return Membership{Node: "n1", Signer: public}, private
}
// Measured on the anchor: a declaration carried a new controller and a new host, the host applied
// it, stood aside for its successor — which cancels the link — and the report of that apply failed
// with "context canceled". The plan waited on it until somebody pushed by hand.
//
// R3 of novox/hq to-be 45 §9, as a unit: the node-engine self-updates during its report, and the
// report still arrives — on the link the apply came over, before it is let go.
func TestAnApplyThatEndsTheLinkIsStillReported(t *testing.T) {
m, key := aMember(t)
declaration := &said{body: signedBy(t, key, []byte(`{"declaration":1}`))}
behind := &said{body: signedBy(t, key, []byte(`{"declaration":2}`))}
l := newQuietLink(declaration)
kept := &keptInMemory{}
ctx, standAside := context.WithCancel(context.Background())
defer standAside()
applied := 0
q := aQueue(m, func(context.Context, []byte, []byte) Report {
applied++
standAside() // the host delivered its successor, and stands aside for it
return Report{Declared: "d1", Applied: []string{"a"}}
}, kept)
worker := runWorker(ctx, q)
done := make(chan error, 1)
go func() { done <- serve(ctx, l, m, q, nil, time.Second) }()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("the link did not end after the apply stood aside")
}
// Something delivered after the host stood aside is not applied by it: the successor will be
// sent it again.
q.Deliver(behind)
<-worker
reports := l.said()
if len(reports) != 1 || reports[0].Declared != "d1" || !reflect.DeepEqual(reports[0].Applied, []string{"a"}) {
t.Fatalf("the apply's report did not reach the mesh: %+v", reports)
}
if !declaration.wasHandled() {
t.Fatal("the declaration was not settled after its report")
}
if applied != 1 || behind.wasHandled() {
t.Fatalf("%d applies, and the declaration behind settled %v: after standing aside nothing "+
"further is applied", applied, behind.wasHandled())
}
if _, ok, _ := kept.Pending(); ok {
t.Fatal("a report the mesh took is still kept to be said again")
}
}
// A report that did not reach the mesh — the host died between applying and publishing, or the
// broker refused it — is said on the next link, before anything else, and exactly as it was made.
func TestAReportLostAfterTheApplyIsSaidOnTheNextLink(t *testing.T) {
m, key := aMember(t)
made := Report{Declared: "d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"}}
kept := &keptInMemory{}
// The apply happens and its report is lost.
first := newQuietLink(&said{body: signedBy(t, key, []byte(`{"declaration":1}`))})
first.refusing = true
ctx, stop := context.WithCancel(context.Background())
q := aQueue(m, func(context.Context, []byte, []byte) Report { stop(); return made }, kept)
worker := runWorker(ctx, q)
if err := serve(ctx, first, m, q, nil, time.Second); err != nil {
t.Fatal(err)
}
<-worker
if len(first.said()) != 0 {
t.Fatal("the broker refused the report and it counted as said")
}
lost, ok, _ := kept.Pending()
if !ok || lost.ReportSequence != 1 {
t.Fatalf("the lost report was not kept as it was made: %+v", lost)
}
// The next host links. It is stopped at once, so all it does is what it does on linking.
next := newQuietLink()
gone, cancel := context.WithCancel(context.Background())
cancel()
never := func(context.Context, []byte, []byte) Report {
t.Fatal("nothing was delivered, and something was applied")
return Report{}
}
if err := serve(gone, next, m, aQueue(m, never, kept), nil, time.Second); err != nil {
t.Fatal(err)
}
reports := next.said()
if len(reports) != 1 || !reflect.DeepEqual(reports[0], lost) {
t.Fatalf("the lost report was not said again as it was made: %+v", reports)
}
if _, ok, _ := kept.Pending(); ok {
t.Fatal("a report said again is still kept")
}
}
// A node that never applied anything, or whose last report was taken, says nothing on linking.
func TestNothingIsSaidAgainWhenNothingWasLost(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
gone, cancel := context.WithCancel(context.Background())
cancel()
if err := serve(gone, l, m, aQueue(m, nil, &keptInMemory{}), nil, time.Second); err != nil {
t.Fatal(err)
}
if err := serve(gone, l, m, aQueue(m, nil, nil), nil, time.Second); err != nil {
t.Fatal(err)
}
if reports := l.said(); len(reports) != 0 {
t.Fatalf("a node with nothing lost reported %+v", reports)
}
}
internal/link/witnessing.go
+1 -1
View File
@@ -48,7 +48,7 @@ func DirectGetSubject(bucket, key string) string {
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
// Asker is the link open now, when there is one and it can ask.
func (q *Queue) Asker() Asker {
func (q *Session) Asker() Asker {
q.init()
q.mu.Lock()
defer q.mu.Unlock()
internal/store/numbers.go
-94
View File
@@ -1,94 +0,0 @@
package store
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
)
// What the node-engine counts about what it says, kept so the counting outlives the process.
//
// **The report sequence must increase across restarts and self-updates** (novox/hq to-be 45 §6). The
// mesh keeps, per machine, the highest report it accepted and refuses an older one; a host that began
// again from one after every restart — and a self-update is a restart — would have every report it
// made refused until it had counted past where its predecessor stopped. So the number is kept beside
// the state, where the successor reads it.
//
// And the count of declarations refused as older than what this node applied (rule 2), which the
// mesh's stale-writer watchdog reads from every report.
// NumbersName is where they live, beside the state.
const NumbersName = "numbers.json"
// NumbersPath is where the numbers live, given where the state lives.
func NumbersPath(statePath string) string {
return filepath.Join(filepath.Dir(statePath), NumbersName)
}
// Numbers is what is kept.
type Numbers struct {
// ReportSequence is the number of the last report this node made.
ReportSequence int64 `json:"report_sequence"`
// RefusedOlder is how many declarations it has refused as older, ever.
RefusedOlder int64 `json:"refused_older"`
}
// ReadNumbers is what was kept, or zero when nothing was — a node that has never reported.
//
// **Unreadable is an error, never zero** (novox/hq ADR 0227, rule 4): read as zero, every report the
// node makes next would be older than the ones it already made, and refused.
func ReadNumbers(path string) (Numbers, error) {
raw, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return Numbers{}, nil
}
if err != nil {
return Numbers{}, err
}
var n Numbers
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&n); err != nil {
return Numbers{}, fmt.Errorf("the numbers kept at %s are unreadable: %w", path, err)
}
if n.ReportSequence < 0 || n.RefusedOlder < 0 {
return Numbers{}, fmt.Errorf("the numbers kept at %s are below zero, which nothing counting writes", path)
}
return n, nil
}
// SaveNumbers keeps them, replacing what was kept, and is on disk when it returns: a number used and
// not kept is one the next host would use again.
func SaveNumbers(path string, n Numbers) error {
raw, err := json.Marshal(n)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
tmp, err := os.CreateTemp(filepath.Dir(path), ".numbers-*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if err := tmp.Chmod(0o600); err != nil {
tmp.Close()
return err
}
if _, err := tmp.Write(raw); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
internal/store/unsaid.go
-76
View File
@@ -1,76 +0,0 @@
package store
import (
"errors"
"os"
"path/filepath"
)
// The report of the last apply of a declaration from the mesh, kept until the mesh has taken it.
//
// **An apply the mesh never heard about is one it waits on for ever.** The controller's release
// plan waits for a machine to report the exact declaration it was sent; a report lost between the
// apply and the publish — a host standing aside for its successor, a crash, a power cut — leaves the
// machine applied and the plan waiting until somebody pushes by hand (novox/hq issue 264). The
// declaration itself is kept (declared.go), and re-applying it would say something new about the
// machine; what was lost is what *that* apply did, so that is what is kept, exactly as it was made,
// and said again once the host is linked.
//
// Opaque bytes here: the report is the link's word, and the store keeps it without reading it.
// UnsaidName is where it lives, beside the state.
const UnsaidName = "unsaid.json"
// UnsaidPath is where the unsaid report lives, given where the state lives.
func UnsaidPath(statePath string) string {
return filepath.Join(filepath.Dir(statePath), UnsaidName)
}
// SaveUnsaid keeps a report the mesh has not taken yet, replacing whatever was kept before: only
// the last apply's report is worth saying again, because the mesh compares against what it sent last.
func SaveUnsaid(path string, report []byte) error {
if len(report) == 0 {
return errors.New("refusing to keep an empty report")
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
tmp, err := os.CreateTemp(filepath.Dir(path), ".unsaid-*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if err := tmp.Chmod(0o600); err != nil {
tmp.Close()
return err
}
if _, err := tmp.Write(report); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
// ReadUnsaid is the report kept unsaid, or nil when there is none — the ordinary case.
func ReadUnsaid(path string) ([]byte, error) {
raw, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return raw, err
}
// ClearUnsaid forgets the kept report, once the mesh has taken it. Absent is already clear.
func ClearUnsaid(path string) error {
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}