diff --git a/cmd/mesh-host/busy_test.go b/cmd/mesh-host/busy_test.go index b1edaea..a7cac9c 100644 --- a/cmd/mesh-host/busy_test.go +++ b/cmd/mesh-host/busy_test.go @@ -5,62 +5,30 @@ import ( "sync/atomic" "testing" "time" - - "github.com/novox/mesh-host/internal/link" ) -// A container waiting because it said busy is news when it starts waiting and when it is replaced, never on -// its output alone (novox/hq ADR 0335): the walk waiting on this machine starts its gate at the report that -// says it replaced, and a reconcile is otherwise silent. -func TestABusyWaitIsSaidWhenItStartsAndWhenItEnds(t *testing.T) { - quiet := link.Report{Outward: []string{"eth0"}} - waiting := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server", - Container: "mesh-build-agent", State: "waiting", Said: "building a"}}} - still := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server", - Container: "mesh-build-agent", State: "waiting", Said: "building b, a minute on"}}} - replaced := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server", - Container: "mesh-build-agent", State: "replaced-when-idle"}}} - if adoptionFingerprint(waiting) == adoptionFingerprint(quiet) { - t.Error("a container starting to wait is not news") - } - if adoptionFingerprint(still) != adoptionFingerprint(waiting) { - t.Error("a wait whose output changed reads as news on every ask") - } - if adoptionFingerprint(replaced) == adoptionFingerprint(waiting) { - t.Error("a container replaced after it waited is not news") - } - if !worthSaying(link.Report{Busy: replaced.Busy}) { - t.Error("a replacement after a wait is not worth saying") - } -} - -// While a container waits, a pass is asked for every so often, not at the reconcile's five minutes; with -// nothing waiting, none (novox/hq ADR 0335 §2). -func TestAWaitIsAskedAgainSoonerThanTheReconcile(t *testing.T) { - if BusyAskEvery >= ReconcileEvery || BusyAskEvery > time.Minute { +// While a container waits, a pass is asked for every so often, not at some long interval; with nothing waiting, none +// (novox/hq ADR 0335 §2, ADR 0360 §11). +func TestAWaitIsAskedAgainSoon(t *testing.T) { + if BusyAskEvery > time.Minute { t.Fatalf("a waiting container is asked every %s", BusyAskEvery) } - var reconciles atomic.Int32 - queue := &link.Queue{Reconcile: func(context.Context) (link.Report, bool) { - reconciles.Add(1) - return link.Report{}, false - }} + var asked atomic.Int32 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go queue.Run(ctx) busyWaiting.Store(false) t.Cleanup(func() { busyWaiting.Store(false) }) - go busyPasses(ctx, queue, 10*time.Millisecond) + go busyPasses(ctx, func() { asked.Add(1) }, 10*time.Millisecond) time.Sleep(100 * time.Millisecond) - if n := reconciles.Load(); n != 0 { + if n := asked.Load(); n != 0 { t.Fatalf("%d passes were asked with nothing waiting", n) } busyWaiting.Store(true) deadline := time.Now().Add(2 * time.Second) - for reconciles.Load() == 0 && time.Now().Before(deadline) { + for asked.Load() == 0 && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) } - if reconciles.Load() == 0 { + if asked.Load() == 0 { t.Fatal("no pass was asked while a container waits") } } diff --git a/cmd/mesh-host/epoch_test.go b/cmd/mesh-host/epoch_test.go deleted file mode 100644 index 56bb0bc..0000000 --- a/cmd/mesh-host/epoch_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "errors" - "os" - "path/filepath" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be -// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node -// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before -// the machine is read or touched, with a report naming what was refused and what is held, and the -// kept declaration left as it was. -func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) { - dir := t.TempDir() - opts := options{state: filepath.Join(dir, "state.json")} - _, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - target := filepath.Join(dir, "a.conf") - declared := func(epoch, sequence int64) store.Declared { - body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence, - "resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}}) - if err != nil { - t.Fatal(err) - } - return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)} - } - held := declared(57, 3) - if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil { - t.Fatal(err) - } - - stale := declared(41, 12) - report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil) - if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) || - report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) || - report.Refused == "" { - t.Fatalf("the refusal does not name what was refused and what is held: %+v", report) - } - if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { - t.Fatal("the refused declaration touched the machine") - } - kept, err := store.ReadDeclared(store.DeclaredPath(opts.state)) - if err != nil || string(kept.Declaration) != string(held.Declaration) { - t.Fatal("the refused declaration replaced what this node kept") - } -} - -// The queue's counts are kept beside the state and read back by the next host; unreadable is an -// error, never zero. -func TestTheNumbersSurviveTheHost(t *testing.T) { - state := filepath.Join(t.TempDir(), "state.json") - if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 { - t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err) - } - if err := numbersBeside(state).Save(41, 2); err != nil { - t.Fatal(err) - } - if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 { - t.Fatalf("read back %d, %d, %v", sequence, refused, err) - } - if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil { - t.Fatal(err) - } - if _, _, err := numbersBeside(state).Read(); err == nil { - t.Fatal("unreadable numbers were read as zero") - } -} diff --git a/cmd/mesh-host/generation_floor_test.go b/cmd/mesh-host/generation_floor_test.go deleted file mode 100644 index 81ec418..0000000 --- a/cmd/mesh-host/generation_floor_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "context" - "testing" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/store" - "github.com/novox/mesh-host/internal/system" -) - -// **The generation applied is kept, and a put-back never lowers it**: after a put-back of generation 5 -// over 7, a send from generation 6 is still older than what this machine applied. -func TestThePutBackDoesNotLowerTheGenerationApplied(t *testing.T) { - built, err := system.For("arch") - if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { - t.Skip("this machine is not one these tests can apply on") - } - was := builtFor - builtFor = "arch" - t.Cleanup(func() { builtFor = was }) - f := machineThatApplied(t, 7) - - newer := f.declared(t, map[string]any{"sequence": 9, "generation": 8}, "docker.conf", f.stays) - if r := applyAndKeep(context.Background(), f.opts, newer.Declaration, &newer, nil, nil); r.Refused != "" { - t.Fatalf("a newer generation was refused: %s", r.Refused) - } - back := f.declared(t, map[string]any{"sequence": 10, "generation": 5, "put_back": true}, "docker.conf", f.stays) - if r := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil); r.Refused != "" { - t.Fatalf("a put-back was refused: %s", r.Refused) - } - state, err := store.Load(f.opts.state) - if err != nil || state.Generation != 8 { - t.Fatalf("the generation applied is %d after a put-back of 5 over 8 (%v)", state.Generation, err) - } - stale := f.declared(t, map[string]any{"sequence": 11, "generation": 7}, "docker.conf", f.stays) - if r := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil); r.OlderGeneration == nil { - t.Fatalf("a send from generation 7 after 8 was applied because a put-back came between: %+v", r) - } -} diff --git a/cmd/mesh-host/generation_test.go b/cmd/mesh-host/generation_test.go deleted file mode 100644 index a9b02f5..0000000 --- a/cmd/mesh-host/generation_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package main - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// The assignment generation a declaration was composed from (novox/hq issue 234). -// -// On 2026-10-04 a declaration newer in sequence than every other, and composed from a view of the -// assignments older than the one this machine had applied, named four fewer modules — and the host, -// ordering by sequence alone, undeclared all four. The sequence orders arrival; it cannot tell a later -// send carrying earlier content. The generation can: the node-engine refuses a declaration composed from -// an older generation than the one it applied, before anything is read from the machine. -// -// These read what the report says through its JSON, as the controller does, so they run against a -// node-engine that has never heard of a generation and fail there for that reason. - -// generationFixture is a machine that applied generation 7: a file the mesh declared, kept in its state, -// and the declaration it kept — signed — that named it. -type generationFixture struct { - opts options - private ed25519.PrivateKey - kept store.Declared - // stays is the file the older generation's declaration does not name. - stays string -} - -func machineThatApplied(t *testing.T, generation int64) generationFixture { - t.Helper() - dir := t.TempDir() - opts := options{state: filepath.Join(dir, "state.json")} - _, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - stays := filepath.Join(dir, "docker.conf") - if err := os.WriteFile(stays, []byte("kept\n"), 0o644); err != nil { - t.Fatal(err) - } - // The state as a node-engine that applied generation 7 keeps it: the file it owns, and the highest - // generation it applied. Written as JSON, so the fixture is the same bytes for any build under test. - state := map[string]any{"resources": []any{map[string]any{"id": "docker.conf", "type": "file", - "origin": store.OriginDeclared, "target": stays}}, "generation": generation} - raw, err := json.Marshal(state) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(opts.state, raw, 0o600); err != nil { - t.Fatal(err) - } - f := generationFixture{opts: opts, private: private, stays: stays} - f.kept = f.declared(t, map[string]any{"sequence": 3, "generation": generation}, "docker.conf", stays) - if err := store.SaveDeclared(store.DeclaredPath(opts.state), f.kept); err != nil { - t.Fatal(err) - } - return f -} - -// declared is a signed declaration with the envelope's keys given and one file. -func (f generationFixture) declared(t *testing.T, envelope map[string]any, id, path string) store.Declared { - t.Helper() - body := map[string]any{"declaration": 1, - "resources": []any{map[string]any{"id": id, "type": "file", "path": path, "content": "x\n"}}} - for k, v := range envelope { - body[k] = v - } - raw, err := json.Marshal(body) - if err != nil { - t.Fatal(err) - } - return store.Declared{Declaration: raw, Signature: ed25519.Sign(f.private, raw)} -} - -// reportJSON is a report as the controller reads it. -func reportJSON(t *testing.T, r link.Report) map[string]any { - t.Helper() - raw, err := json.Marshal(r) - if err != nil { - t.Fatal(err) - } - var out map[string]any - if err := json.Unmarshal(raw, &out); err != nil { - t.Fatal(err) - } - return out -} - -// **A newer sequence from an older generation is refused, and nothing is undeclared** (issue 234). -func TestANewerSequenceFromAnOlderGenerationIsRefusedAndNothingIsUndeclared(t *testing.T) { - f := machineThatApplied(t, 7) - other := filepath.Join(filepath.Dir(f.opts.state), "sudo.conf") - stale := f.declared(t, map[string]any{"sequence": 9, "generation": 6}, "sudo.conf", other) - - report := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil) - said := reportJSON(t, report) - older, _ := said["older_generation"].(map[string]any) - if older == nil || older["generation"] != float64(6) || older["applied"] != float64(7) { - t.Fatalf("the refusal does not name the generation refused and the one applied: %v", said) - } - if report.Refused == "" || !strings.Contains(report.Refused, "generation 6") || - !strings.Contains(report.Refused, "generation 7") { - t.Errorf("the refusal is not said in words naming both generations: %q", report.Refused) - } - if report.Declared != digestOf(stale.Declaration) || report.Sequence != 9 { - t.Errorf("the refusal does not name the declaration refused, so the mesh cannot name who sent it: %+v", report) - } - if _, err := os.Stat(f.stays); err != nil { - t.Fatalf("a refused declaration undeclared what the machine applied: %v", err) - } - if _, err := os.Stat(other); err == nil { - t.Error("the refused declaration touched the machine") - } - state, err := store.Load(f.opts.state) - if err != nil { - t.Fatal(err) - } - if _, owned := state.Find("docker.conf"); !owned { - t.Error("the refused declaration took a resource out of what this machine owns") - } - kept, err := store.ReadDeclared(store.DeclaredPath(f.opts.state)) - if err != nil || string(kept.Declaration) != string(f.kept.Declaration) { - t.Error("the refused declaration replaced what this node kept, so the next reconcile would apply it") - } -} - -// refusedForItsGeneration says a report refused its declaration for the generation it came from. -func refusedForItsGeneration(t *testing.T, r link.Report) bool { - t.Helper() - return reportJSON(t, r)["older_generation"] != nil || strings.Contains(r.Refused, "generation") -} - -// refusedAsUnread says a report refused its declaration as one this node-engine cannot read. -func refusedAsUnread(r link.Report) bool { return strings.Contains(r.Refused, "not a declaration") } - -// **A gate's put-back is applied, whatever generation it carries**: it carries the generation of what it -// puts back, and says it is a put-back (issue 234). -func TestAPutBackFromAnOlderGenerationIsNotRefused(t *testing.T) { - f := machineThatApplied(t, 7) - back := f.declared(t, map[string]any{"sequence": 9, "generation": 5, "put_back": true}, "docker.conf", f.stays) - report := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil) - if refusedForItsGeneration(t, report) || refusedAsUnread(report) { - t.Fatalf("a put-back was refused: %s", report.Refused) - } -} - -// **A declaration that carries no generation is applied as it always was**: an older controller sends -// none, and a node-engine that refused it would strand every machine the moment the controller is older. -func TestADeclarationWithoutAGenerationIsNotRefusedForOne(t *testing.T) { - f := machineThatApplied(t, 7) - older := f.declared(t, map[string]any{"sequence": 9}, "docker.conf", f.stays) - report := applyAndKeep(context.Background(), f.opts, older.Declaration, &older, nil, nil) - if refusedForItsGeneration(t, report) || refusedAsUnread(report) { - t.Fatalf("a declaration without a generation was refused: %s", report.Refused) - } - // And the same generation again is no older. - same := f.declared(t, map[string]any{"sequence": 10, "generation": 7}, "docker.conf", f.stays) - report = applyAndKeep(context.Background(), f.opts, same.Declaration, &same, nil, nil) - if refusedForItsGeneration(t, report) || refusedAsUnread(report) { - t.Fatalf("a declaration of the generation applied was refused: %s", report.Refused) - } -} diff --git a/cmd/mesh-host/local_network_test.go b/cmd/mesh-host/local_network_test.go deleted file mode 100644 index 3a7af99..0000000 --- a/cmd/mesh-host/local_network_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/outward" - "github.com/novox/mesh-host/internal/profile" -) - -func reportOn(networks ...outward.Network) link.Report { - return link.Report{Outward: []string{"eth0"}, - Profile: profileAsReported(profile.Profile{Machine: &profile.Machine{Networks: networks}})} -} - -// **A change of the networks a machine is attached to is said unasked** (novox/hq ADR 0340 §7): a port -// reached from the local network is admitted from them, so a laptop that joins another network is said at -// the next reconcile, not at the next push — and the same networks again are not news. -func TestAChangedNetworkIsSaidUnasked(t *testing.T) { - w := &adoptionWatch{} - home := reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"}) - w.said(home) - if w.differs(reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})) { - t.Fatal("the same networks read as a change, so the machine would speak on every reconcile") - } - if !w.differs(reportOn(outward.Network{Link: "eth0", Network: "10.20.0.0/16"})) { - t.Fatal("a machine that moved to another network does not say so until it is pushed to") - } -} diff --git a/cmd/mesh-host/machine_refresh_test.go b/cmd/mesh-host/machine_refresh_test.go deleted file mode 100644 index 91c2dda..0000000 --- a/cmd/mesh-host/machine_refresh_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-host/internal/link" -) - -// Defends novox/hq issue 160: a machine nobody pushed to still says what it is made of once a day, so the -// newest of it the mesh holds is no older than a day. Before, the reconcile said it only with something -// else (a hold, a firewall, an outward link changed), and a converged machine's picture was as old as the -// last delivery to it. -func TestAMachineNobodyPushedToSaysWhatItIsOnceADay(t *testing.T) { - at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) - w := &adoptionWatch{now: func() time.Time { return at }} - quiet := link.Report{Outward: []string{"wlan0"}, - Profile: map[string]any{"machine": map[string]any{"observed_at": at}}} - - if !w.news(quiet) { - t.Fatal("the first reconcile after the engine started did not say what the machine is") - } - w.said(quiet) - - for _, after := range []time.Duration{5 * time.Minute, 12 * time.Hour, 22*time.Hour + 55*time.Minute} { - at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(after) - if w.news(quiet) { - t.Errorf("%s after the mesh heard it, an unchanged reconcile was said again", after) - } - } - // The reconcile that falls past 23 hours says it, so it reaches the mesh inside the day. - at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(23*time.Hour + 5*time.Minute) - if !w.news(quiet) { - t.Fatal("a day on, the reconcile did not say what the machine is") - } - w.said(quiet) - at = at.Add(5 * time.Minute) - if w.news(quiet) { - t.Error("once said, it was said again at the next reconcile") - } -} - -// A delivery's account carries the profile too: heard, it starts the day again. -func TestADeliveryHeardStartsTheDayAgain(t *testing.T) { - at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) - w := &adoptionWatch{now: func() time.Time { return at }} - report := link.Report{Outward: []string{"eth0"}, Profile: map[string]any{"capabilities": []any{}}} - w.said(report) - at = at.Add(20 * time.Hour) - w.said(report) // a delivery's apply - at = at.Add(20 * time.Hour) - if w.news(report) { - t.Error("said again unasked 20 hours after a delivery told the mesh") - } -} - -// A refusal carries nothing about the machine, so it is never said for being a day old. -func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) { - at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) - w := &adoptionWatch{now: func() time.Time { return at }} - if w.machineDue(link.Report{Refused: "no"}) { - t.Error("a refusal was due for what the machine is made of") - } -} - -// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic -// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses -// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first -// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no -// monotonic reading, and judge the day as the wall clock says it. -func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) { - before := time.Now() // carries a monotonic reading - asleep := before.Add(3 * time.Second) - if asleep.Sub(before) != 3*time.Second { - t.Fatal("the monotonic reading is not what this test simulates with") - } - calls := 0 - w := &adoptionWatch{now: func() time.Time { - calls++ - if calls == 1 { - return before - } - // The wall clock a day on, as a machine's clock reads after waking. - return asleep.Round(0).Add(24 * time.Hour) - }} - report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}} - w.said(report) - if strings.Contains(w.machineSaid.String(), "m=") { - t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid) - } - if !w.news(report) { - t.Error("a day of wall time on, a laptop that slept did not say what it is") - } -} - -// And the watch's own clock never carries a monotonic reading. -func TestTheWatchClockIsWallTime(t *testing.T) { - w := &adoptionWatch{} - if s := w.clock().String(); strings.Contains(s, "m=") { - t.Errorf("the watch's clock carries a monotonic reading: %s", s) - } - w = &adoptionWatch{now: time.Now} - if s := w.clock().String(); strings.Contains(s, "m=") { - t.Errorf("a given clock's time kept its monotonic reading: %s", s) - } -} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index a0c4e1b..6572167 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -105,8 +105,6 @@ reach, run as root at the machine. profile what this machine can be asked to do inventory what this machine is, and what it holds apply FILE make this machine match a declaration from a file - reconcile make this machine match what the mesh last told it — or, before any - mesh has, the bundle this host carries bundle show what this host carries key make this machine's tunnel key, or read the one it made, and print the public half: what its join token is issued for (novox/hq ADR 0169) @@ -126,10 +124,9 @@ reach, run as root at the machine. --state where this node keeps what it knows (default /var/lib/mesh-host/state.json) --dry-run say what applying would change, and change nothing -Both apply and reconcile say what they would change before changing anything, and refuse a -declaration for the other mode than this node is in, or one older than what the mesh last said. - -It connects to nothing and listens on nothing. What it applies comes from a file. +Apply says what it would change before changing anything, and refuses a declaration for the other mode than this +node is in. It is for a machine the mesh has not spoken to. Once enrolled, the node follows its wants (novox/hq ADR +0539): it applies a module from the composition its want names, and no whole declaration. ` func main() { @@ -286,13 +283,6 @@ func run(ctx context.Context, command string, opts options) error { } return runApply(ctx, opts, d, raw, fromFile) - case "reconcile": - d, raw, from, err := reconcileSource(opts) - if err != nil { - return err - } - return runApply(ctx, opts, d, raw, from) - case "bundle": if bundle.IsEmpty(builtFor) { fmt.Println("this host carries no bundle") @@ -412,58 +402,12 @@ type provenance int const ( // fromFile is `apply FILE`: handed to the host by someone already running it as root. fromFile provenance = iota - // fromBundle is `reconcile` on a machine the mesh has told nothing yet: the bundle carried in + // fromBundle is genesis on a machine the mesh has told nothing yet: the bundle carried in // the binary, the first node's path before its mesh is up (novox/hq ADR 0004). fromBundle - // fromDeclared is `reconcile` on a node the mesh has spoken to: what it last said, kept - // signed beside the state (declared.json), verified again before it is applied. - fromDeclared ) -// reconcileSource is which declaration `reconcile` holds this machine to. -// -// **What the mesh last said, never the bundle, once the mesh has said anything** (novox/hq issue -// 104). The bundle is right at genesis and stale a minute later — genesis rewrites it for the -// machine before applying it, and every declaration since came from the controller — and on an -// adopted node it is a converged declaration for a machine that is not converged. A node that -// has been told something and cannot prove it is the mesh's is refused, with the reason; the -// bundle is not applied in its place. -func reconcileSource(opts options) (*declaration.Declaration, []byte, provenance, error) { - path := store.DeclaredPath(opts.state) - _, err := store.ReadDeclared(path) - switch { - case err == nil: - mine, err := identity.Load(identity.Path(opts.state)) - if err != nil { - return nil, nil, 0, fmt.Errorf("this node was told a declaration by the mesh, kept at %s, "+ - "and cannot prove it is the mesh's: %w\n\nIt is not applied unproven, and the bundle "+ - "this host carries is not applied in its place: that was consumed at genesis", path, err) - } - raw, err := store.LoadDeclared(path, mine.Membership.Signer) - if err != nil { - return nil, nil, 0, fmt.Errorf("%w\n\nThe bundle this host carries is not applied in its "+ - "place: that was consumed at genesis", err) - } - d, err := declaration.Parse(raw) - if err != nil { - return nil, nil, 0, err - } - return d, raw, fromDeclared, nil - case errors.Is(err, store.ErrNothingDeclared): - d, err := bundle.Load(builtFor) - if err != nil { - return nil, nil, 0, err - } - return d, bundle.Raw(builtFor), fromBundle, nil - default: - return nil, nil, 0, err - } -} - func (p provenance) origin() string { - if p == fromDeclared { - return store.OriginDeclared - } // A file handed to the host is applied as the bundle is: what it puts here is invisible to // the removal pass of a declaration that later arrives from the mesh (novox/hq issue 010). return store.OriginCarried @@ -473,8 +417,6 @@ func (p provenance) name(opts options) string { switch p { case fromBundle: return "the carried bundle" - case fromDeclared: - return "what the mesh last told this node (" + store.DeclaredName + ")" } return opts.file } @@ -499,10 +441,8 @@ func short(digest string) string { // them again — and everything the mesh declared read as no longer declared and removed. Even the // very declaration the mesh last sent, applied from a file, would plan to remove the foundation. // `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so. -func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, order link.Order) error { +func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error { switch from { - case fromDeclared: - return refuseOlder(kept, keptErr, order) case fromBundle: if known.Genesis == nil || known.Genesis.Digest == digest { return nil @@ -536,8 +476,8 @@ func refuseStale(known store.State, kept store.Declared, keptErr error, digest s return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+ "node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+ "recorded as this machine's own, which the mesh could then never remove, and what the mesh "+ - "declared read as no longer declared — so no file is applied here: `reconcile` applies what "+ - "the mesh last said, and `push` from the controller changes it", + "declared read as no longer declared — so no file is applied here: the node follows its wants "+ + "(novox/hq ADR 0539)", short(last), store.DeclaredName, what) } @@ -585,13 +525,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the // note, or to the kept declaration when the note predates it. kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)) - if from == fromDeclared { - if known.Mode != "" && known.Mode != apply.ModeOf(d) { - fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n", - known.Mode, apply.ModeOf(d)) - } - known.Mode = apply.ModeOf(d) - } else if known.Mode == "" && keptErr == nil { + if known.Mode == "" && keptErr == nil { if last, err := declaration.Parse(kept.Declaration); err == nil { known.Mode = apply.ModeOf(last) } @@ -599,14 +533,9 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw if err := apply.CheckMode(known, d); err != nil { return err } - if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil { + if err := refuseStale(known, kept, keptErr, digest, from); err != nil { return err } - // The generation, as the link's deliveries are held to it (novox/hq issue 234): one composed from an - // older view of the assignments than this machine applied would undeclare what the mesh still assigns. - if refused := olderGeneration(known, d); refused != nil { - return errors.New(olderGenerationWords(*refused)) - } // A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has // no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty. @@ -674,7 +603,6 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // What this apply settles about the node, whichever way it went. The mode is what the // declaration said and the check above agreed with; the bundle, once applied, is consumed. updated.Mode = apply.ModeOf(d) - updated.Generation = appliedGeneration(known, d) // raised, never lowered (novox/hq issue 234) if from == fromBundle { updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()} } @@ -1179,8 +1107,9 @@ func runLink(ctx context.Context, opts options) error { Signer: mine.Membership.Signer, } - applier := func(ctx context.Context, raw, signature []byte) link.Report { - report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say) + // What follows an apply of a module (the node agent's, novox/hq ADR 0539): a membership it delivered for another bus + // is adopted, and a newer node-engine it placed makes this one stand aside. + afterApply := func() { // **A declaration may carry this machine's membership for another bus.** It arrives as a // sealed file like any secret, and is read after the rest has applied so the bus it names is // standing before this machine leaves the one it is on (novox/hq design 28, task 5.2). @@ -1201,35 +1130,16 @@ func runLink(ctx context.Context, opts options) error { stoodAside = true standAside() } - return report } - // Two things at once, and the second is what makes disconnection ordinary. The link brings - // new declarations; the reconcile holds the machine in the last one whether the link is up or - // not. A laptop shut for a week comes back and reconciles — it does not come back and ask what - // it is (novox/hq ADR 0004). - // - // **Both only enqueue** (novox/hq to-be 45 §6). One worker applies, the newest declaration held - // when it starts, once, and makes one report; a reconcile due while a delivery waits is that - // delivery's apply. Reports a reconcile has to make unasked — what an adopted node holds changed, - // its firewall, its outward links — are said by the same worker, in the order they are made - // (novox/hq ADR 0100, issue 267). - watch := &adoptionWatch{} + // The node follows its wants; it is never sent a declaration (novox/hq ADR 0539 §9). What the session does is hold + // the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the + // controller asks. proof := &proving{statePath: opts.state, version: runningVersion(), say: say} - queue := &link.Queue{ + queue := &link.Session{ Membership: membership, - Apply: applier, - Reconcile: reconcileKept(opts, mine.Membership.Signer, sched, say), - News: watch.news, - // Counted as said only once the broker has taken it: queued and lost — the link down, the - // publish refused — the change would never be said again (novox/hq ADR 0100). - // And the first account under this build is what proves it to the launcher (to-be 45 §8). - Heard: func(r link.Report) { - watch.said(r) - proof.heard(r) - }, - Unsaid: unsaidBeside(opts.state), - Numbers: numbersBeside(opts.state), + // And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8). + HaveWritten: proof.haveWritten, // And the operator's hand-over of a directory used as found, carried by the controller from the // terminal (novox/hq issue 356): judged and recorded here, applied by the next apply. HandOver: handOverFrom(opts.state), @@ -1248,6 +1158,10 @@ func runLink(ctx context.Context, opts options) error { Say: say, Timeout: opts.timeout, } + queue.OnLink = func(ctx context.Context) { + sayFacts(ctx, opts, mine.Membership.Signer, queue, say) + afterApply() + } // And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3, // issue 676): the vault is asked over this queue's link, so the queue is named once it exists. queue.MoveSecret = moveSecretFrom(opts.state, queue) @@ -1258,25 +1172,6 @@ func runLink(ctx context.Context, opts options) error { }, func(verb string, args any) (json.RawMessage, error) { return queue.AskVault(ctx, verb, args) }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo) - worker := make(chan struct{}) - go func() { - defer close(worker) - queue.Run(aside) - }() - // **A host starting is a reason to reconcile** (to-be 45 §6: the self-update hand-over is one of - // the four): its scheduled steps are armed from the declaration the node kept by the first apply, - // and a successor that waited five minutes for it left them unarmed for five. - // - // **A host on trial says its account whether or not it is news** (to-be 45 §8): the launcher - // waits for this build to report its declaration, and a converged machine with nothing new to - // say would otherwise not say anything until the mesh next sent it something. - if unproved(opts.state, runningVersion()) { - queue.ReportAsked() - } else { - queue.ReconcileDue() - } - go holdTheMachine(ctx, queue) - go busyPasses(ctx, queue, BusyAskEvery) // And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq // ADR 0240) — between applies, which is when a container crash-loops. if judging != nil { @@ -1301,9 +1196,6 @@ func runLink(ctx context.Context, opts options) error { go serveFilling(aside, opts, mine.Membership.Signer, say) held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx)) - // The act in hand finishes and is kept before this process exits: an apply that stood aside with - // no link open has its report only in the unsaid store, and only once the worker has written it. - <-worker // **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back // before it has run once. The context this returns on was cancelled deliberately, so its error // is not a fault to report. @@ -1313,241 +1205,6 @@ func runLink(ctx context.Context, opts options) error { return held } -// unsaidFile keeps the last apply's report beside the node's state until the mesh has taken it -// (novox/hq issue 264), so a report lost when this host stood aside for its successor — or died -// between applying and publishing — is said by whichever host runs next, once it is linked. -type unsaidFile struct{ path string } - -func unsaidBeside(statePath string) unsaidFile { return unsaidFile{path: store.UnsaidPath(statePath)} } - -func (u unsaidFile) Keep(r link.Report) error { - // A report naming no declaration is one the mesh cannot match to what it sent, and keeping it - // would only re-say an older apply after a newer one was refused. - if r.Declared == "" { - return store.ClearUnsaid(u.path) - } - body, err := json.Marshal(r) - if err != nil { - return err - } - return store.SaveUnsaid(u.path, body) -} - -func (u unsaidFile) Said(declared string) error { - kept, ok, err := u.Pending() - if err != nil || !ok || kept.Declared != declared { - return err - } - return store.ClearUnsaid(u.path) -} - -func (u unsaidFile) Pending() (link.Report, bool, error) { - raw, err := store.ReadUnsaid(u.path) - if err != nil || raw == nil { - return link.Report{}, false, err - } - var r link.Report - if err := json.Unmarshal(raw, &r); err != nil { - // Unreadable is forgotten rather than kept for ever: it can never be said. - _ = store.ClearUnsaid(u.path) - return link.Report{}, false, fmt.Errorf("the kept report at %s is unreadable, and is dropped: %w", u.path, err) - } - return r, r.Declared != "", nil -} - -// numbersFile keeps the queue's report sequence and its count of declarations refused as older beside -// the node's state (novox/hq to-be 45 §6), where a successor reads them and goes on from there. -type numbersFile struct{ path string } - -func numbersBeside(statePath string) numbersFile { - return numbersFile{path: store.NumbersPath(statePath)} -} - -func (n numbersFile) Read() (int64, int64, error) { - kept, err := store.ReadNumbers(n.path) - return kept.ReportSequence, kept.RefusedOlder, err -} - -func (n numbersFile) Save(reportSequence, refusedOlder int64) error { - return store.SaveNumbers(n.path, store.Numbers{ReportSequence: reportSequence, RefusedOlder: refusedOlder}) -} - -// adoptionWatch remembers what the node last said about what it holds and its firewall, so a -// reconcile speaks unasked only when that changed. -type adoptionWatch struct { - mu sync.Mutex - last string - // faulty is whether the last report the mesh heard said an apply refused or failed (novox/hq issue 233). - faulty bool - // machineSaid is when the mesh last heard what this machine is made of (novox/hq issue 160), and now the - // clock it is judged by (time.Now when nil). - machineSaid time.Time - now func() time.Time -} - -// machineEvery is how old what a machine said of itself may grow before a reconcile says it again unasked -// (novox/hq issue 160): a machine nobody pushed to still says it once a day, so no fact the mesh holds of it -// is older than a day. Under a day by more than one reconcile interval, so the reconcile that says it falls -// inside the day rather than just past it. -const machineEvery = 23 * time.Hour - -// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's -// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept -// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day. -func (w *adoptionWatch) clock() time.Time { - if w.now != nil { - return w.now().Round(0) - } - return time.Now().Round(0) -} - -// machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago -// or more. A refusal carries none, so it never is. -func (w *adoptionWatch) machineDue(r link.Report) bool { - return r.Refused == "" && len(r.Profile) > 0 && w.clock().Sub(w.machineSaid) >= machineEvery -} - -// fingerprint is what a report says about adoption: each hold and whether it changed, the -// firewall, and what is reachable on the machine — which only an adopted node reports, and which -// is what the controller previews a flip from, so a port that opens or closes between deliveries -// must reach it too (novox/hq ADR 0100). -func adoptionFingerprint(r link.Report) string { - // **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR - // 0140). The mesh composes no filter for a machine that has not said, so a machine whose links - // changed — or which has only just learnt to say — has to say so without being asked. Left out, - // it could only speak when a declaration arrived, and a declaration cannot be composed until it - // has spoken: a machine waiting for a push that is waiting for the machine. - parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")} - // **And the networks those links are on** (novox/hq ADR 0340 §7): a port reached from the local - // network is admitted from them, so a laptop that joins another network, or a lease that moves, is - // said at the next reconcile rather than at the next push or the next day. - if networks := attachedNetworksOf(r); networks != "" { - parts = append(parts, "networks="+networks) - } - // And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk - // waiting on this machine starts its gate at the report that says it replaced. Not its output, which - // may change on every ask. - for _, b := range r.Busy { - parts = append(parts, "busy "+b.ID+"="+b.State) - } - for _, h := range r.Held { - parts = append(parts, "held "+h.ID+"="+h.Changed) - } - // And what filters the machine, with the found firewall's state (novox/hq ADR 0168): a rule the - // operator removes between declarations, or a front end enabled again, is said at the next - // reconcile rather than at the next push. - for _, f := range r.Filters { - parts = append(parts, "filter "+f.Owner+" "+f.Where+" "+f.Refuses) - } - // And a witness's verdict (to-be 45 §8): one reached or one ended is said at the next reconcile. - for _, v := range r.Rollbacks { - parts = append(parts, fmt.Sprintf("rollback %s %s %s %s", v.Component, v.From, v.To, v.Outcome)) - } - // **And how the apply went** (novox/hq issue 233): refused, clean, or which resources it did not apply. A - // reconcile is the only apply a machine makes between deliveries, so a machine whose package manager stopped - // answering, or answers again, says so at the next reconcile — otherwise the mesh's record of what it did - // stays whatever the last delivery left, and a fault that began or ended between deliveries is never seen. - // Not the words: an error that carries a time would read as new on every reconcile. - parts = append(parts, applyState(r)) - if r.FoundFirewall != nil { - parts = append(parts, fmt.Sprintf("found-firewall %s active=%v retired-by=%s", r.FoundFirewall.Kind, - r.FoundFirewall.Active, r.FoundFirewall.RetiredBy)) - } - for _, reach := range r.Reachable { - parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address, - reach.Port, reach.By, reach.Published, reach.ContainerPort)) - } - sort.Strings(parts[2:]) - return strings.Join(parts, "\n") -} - -// attachedNetworksOf is the networks a report's profile says the machine is attached to, as one line: the -// part of what the machine is made of that the filter is composed from (novox/hq ADR 0340). -func attachedNetworksOf(r link.Report) string { - machine, ok := r.Profile["machine"].(map[string]any) - if !ok { - return "" - } - networks, ok := machine["networks"].([]any) - if !ok { - return "" - } - said := make([]string, 0, len(networks)) - for _, n := range networks { - if entry, ok := n.(map[string]any); ok { - said = append(said, fmt.Sprintf("%v %v", entry["link"], entry["network"])) - } - } - return strings.Join(said, ",") -} - -// applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not -// apply and which keys its failure is said under. -func applyState(r link.Report) string { - switch { - case r.Refused != "": - return "apply refused" - case len(r.Failed) == 0: - return "apply clean" - } - keys := make([]string, 0, len(r.Failed)) - for k := range r.Failed { - keys = append(keys, k) - } - sort.Strings(keys) - return "apply failed " + strings.Join(keys, ",") + " unapplied " + strings.Join(r.Unapplied, ",") -} - -// faulty is whether a report says an apply refused or failed. -func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 } - -// news is whether a reconcile's report is worth saying unasked: what the machine is made of last reached the -// mesh a day ago or more (novox/hq issue 160); or it says something the last one the mesh heard -// did not, and it is about the machine (worthSaying) — or about how the apply went, refused or failed now, or -// clean after the last one the mesh heard was not (novox/hq issue 233). A refusal says nothing about the -// machine, but that the machine refuses is what the mesh raises a condition from, and a clean apply after it is -// what clears one. -func (w *adoptionWatch) news(r link.Report) bool { - w.mu.Lock() - defer w.mu.Unlock() - // What the machine is made of, once a day whatever else changed (novox/hq issue 160). - if w.machineDue(r) { - return true - } - if adoptionFingerprint(r) == w.last { - return false - } - return worthSaying(r) || faulty(r) || w.faulty -} - -// differs says whether a report says anything the last one that went out did not. It records -// nothing: what was said is what reached the mesh, not what was written down to send. -func (w *adoptionWatch) differs(r link.Report) bool { - w.mu.Lock() - defer w.mu.Unlock() - return adoptionFingerprint(r) != w.last -} - -// said records a report the mesh has actually been told. -func (w *adoptionWatch) said(r link.Report) { - w.mu.Lock() - defer w.mu.Unlock() - w.last = adoptionFingerprint(r) - w.faulty = faulty(r) - if len(r.Profile) > 0 { - w.machineSaid = w.clock() - } -} - -// changed is differs and said together, for a report published as it is made. -func (w *adoptionWatch) changed(r link.Report) bool { - if !w.differs(r) { - return false - } - w.said(r) - return true -} - // rousedBySignal is the machine telling this process that its link is probably stale. // // **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would @@ -1586,14 +1243,6 @@ func rousedBySignal(ctx context.Context) link.Roused { return out } -// ReconcileEvery is how often a node re-applies what it was last told. -// -// Not driven by the link. Changes are pushed, so this is not polling for them — it is the answer -// to a machine drifting: a file edited by hand, a container stopped by somebody, a service that -// died. A node that only acted when told would hold its state exactly until something else -// changed it, and then for ever. -const ReconcileEvery = 5 * time.Minute - // judging is the serving host's liveness judge (novox/hq ADR 0240); nil in a one-shot command, which // reports no health, and in a test. var judging *liveness.Judge @@ -1690,7 +1339,7 @@ const ( // the bus when a state changed, again every minute while one is not healthy, and every five minutes // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it. -func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) { +func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) { ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() var lastSaid time.Time @@ -1911,22 +1560,6 @@ func orNothing(s, unless string) string { return s } -// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq -// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is -// waiting is that delivery's apply. -func holdTheMachine(ctx context.Context, queue *link.Queue) { - ticker := time.NewTicker(ReconcileEvery) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - queue.ReconcileDue() - } - } -} - // busyWaiting is whether the last apply left a container running because it said busy (novox/hq ADR 0335). var busyWaiting atomic.Bool @@ -1935,10 +1568,9 @@ var busyWaiting atomic.Bool // within the reconcile's five minutes — and the bound of 15 minutes is met within this of its end. const BusyAskEvery = 20 * time.Second -// busyPasses asks for a pass every so often while a container waits because it said busy. **It asks; it -// does not apply**, as holdTheMachine: the queue's worker does, when its turn comes, so a pass never holds -// the machine for the wait, and a delivery waiting is that pass's apply. -func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) { +// busyPasses asks for a pass every so often while a container waits because it said busy (ADR 0360 §11). **It asks; it +// does not apply**: the node agent does, when its turn comes, so a pass never holds the machine for the wait. +func busyPasses(ctx context.Context, ask func(), every time.Duration) { ticker := time.NewTicker(every) defer ticker.Stop() for { @@ -1947,66 +1579,12 @@ func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) { return case <-ticker.C: if busyWaiting.Load() { - queue.ReconcileDue() + ask() } } } } -// reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it -// was last told, read once it is this act's turn, and say what did not go. -// -// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or its firewall -// changed, because that is how a predecessor still writing is caught (novox/hq ADR 0100); and on any -// node when it can say which links face outside (ADR 0140) — a converged node holds nothing and found -// no firewall, so without that it could speak only in reply to a declaration, while the mesh composes -// no declaration for a node that has not said which links face outside. Whether a report is news is -// the queue's News (the watch's news), so an unchanged answer costs one comparison every reconcile and -// nothing on the bus. And on any node when how the apply went changed — refused, clean, or which resources it -// did not apply — since that is what the mesh raises a machine that does not apply from, and clears it by -// (novox/hq issue 233). -func reconcileKept(opts options, signer ed25519.PublicKey, sched *apply.Scheduler, - say link.Announce) link.Reconcile { - return func(ctx context.Context) (link.Report, bool) { - report, err := reapplyKept(ctx, opts, signer, sched, say) - if errors.Is(err, store.ErrNothingDeclared) { - // Nothing to hold this machine to yet. Ordinary on a node that has enrolled and not - // been assigned anything. - return link.Report{}, false - } - if err != nil { - say("cannot re-apply what this node was told: " + err.Error()) - return link.Report{}, false - } - switch { - case report.Refused != "": - say("what this node was last told no longer applies: " + report.Refused) - case len(report.Failed) > 0: - say(fmt.Sprintf("holding this machine: %d applied, and %v", len(report.Applied), report.Failed)) - } - return report, true - } -} - -// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked. -// -// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline -// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a -// converged node — which holds nothing and found no firewall — could speak only in reply to a -// declaration, while the mesh composes no declaration for a node that has not said which links face -// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them. -// -// A refused report says nothing about the machine, so it is not news; the refusal is said on the -// console where a person reads it. -func worthSaying(report link.Report) bool { - if report.Refused != "" { - return false - } - return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0 || - len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0 || - len(report.Rollbacks) > 0 || len(report.Busy) > 0 -} - // announceOr is what the apply writes its detail with, given what the caller has to say things with. // // **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply @@ -2043,25 +1621,6 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D return applyAndKeepHeld(ctx, opts, raw, signed, sched, say) } -// reapplyKept re-applies what this node was last told, **read once it is this apply's turn**. -// -// Read before waiting, it was whatever had been kept when the timer fired — and a declaration -// the link was applying at that moment is kept only once its apply ends. The reconcile then waited -// for that apply and applied the older declaration over it: a module assigned a second earlier was -// given back on the spot, and the report it sent named a declaration the mesh no longer recorded -// as sent, so a plan waiting on this machine waited for a report that could not come (novox/hq -// issues 257, 261). Read under the lock, it is always the latest the mesh said. -func reapplyKept(ctx context.Context, opts options, signer ed25519.PublicKey, - sched *apply.Scheduler, say link.Announce) (link.Report, error) { - applying.Lock() - defer applying.Unlock() - declared, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) - if err != nil { - return link.Report{}, err - } - return applyAndKeepHeld(ctx, opts, declared, nil, sched, say), nil -} - // applyAndKeepHeld is applyAndKeep for a caller already holding applying. func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *store.Declared, sched *apply.Scheduler, say link.Announce) link.Report { @@ -2075,36 +1634,10 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto if err != nil { return link.Report{Refused: err.Error()} } - order := link.Order{Epoch: declared.Epoch, Sequence: declared.Sequence} - - // **Older than what this node applied is refused, before anything is read from the machine** - // (novox/hq to-be 45 §6, ADR 0227 rule 2): a controller that lost its lease and goes on sending. - // Read under the lock, so what it is compared with is what the last apply kept. Only a delivery: - // what a reconcile applies is what was kept. - if signed != nil { - kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)) - if held, older := olderThanKept(kept, keptErr, order); older { - return link.Report{Declared: digestOf(raw), Order: order, OlderThan: &held, - Refused: fmt.Sprintf("this declaration was sent under lease epoch %d, sequence %d, and this "+ - "node has applied epoch %d, sequence %d — from the controller holding the lease now. "+ - "Refused whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)} - } - } - known, err := store.Load(opts.state) if err != nil { return link.Report{Refused: err.Error()} } - // **Composed from an older view of the assignments than this machine applied is refused, before - // anything is read from the machine** (novox/hq issue 234). A newer sequence carrying an older view - // undeclared four assigned modules on 2026-10-04; the sequence cannot tell it, the generation can. - // Only a delivery: what a reconcile applies is what was kept. - if signed != nil { - if refused := olderGeneration(known, declared); refused != nil { - return link.Report{Declared: digestOf(raw), Order: order, OlderGeneration: refused, - Refused: olderGenerationWords(*refused)} - } - } // A declaration from the mesh is the controller's word on the node's mode — the flip arrives // as exactly that, the first converged declaration after adopted ones — and becomes the // record. So does what this node kept: it is signed by the mesh and verified on load, where @@ -2124,8 +1657,7 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // 0486 §1, issue 1695): one object missing or differing refuses the declaration whole. The signed bytes are what is // kept; only what is applied has the content in place of the reference. if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil { - return link.Report{Declared: digestOf(raw), Order: order, - Refused: "a file of this declaration could not be had by its digest: " + err.Error()} + return link.Report{Refused: "a file of this declaration could not be had by its digest: " + err.Error()} } built, err := system.For(builtFor) @@ -2169,8 +1701,6 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // The mode the mesh said, recorded whichever way the apply went: the declaration is kept // either way, and the node is held to it from the next reconcile (novox/hq ADR 0100). updated.Mode = apply.ModeOf(declared) - // And the generation it came from, the same way, raised and never lowered (novox/hq issue 234). - updated.Generation = appliedGeneration(known, declared) // Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's // rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it @@ -2204,7 +1734,44 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto sched.Sync(declared, held) } - report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(), + report := factsOf(ctx, opts, declared, updated, outcome, held) + // Kept whichever way it went, so a node that is disconnected next minute still knows what it + // was told. Saved after applying rather than before: what is kept is what this node acted on. + var before []string + if signed != nil { + // The modules the declaration kept until now held, so a have key written for one the new declaration no longer + // holds is written removed even by a node-engine that started since (issue 1369). + if kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)); keptErr == nil { + if old, parseErr := declaration.ParseTrusted(kept.Declaration); parseErr == nil { + for _, f := range moduleFacts(old, apply.Report{}, nil) { + before = append(before, f.Module) + } + } + } + if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil { + report.Failed = map[string]string{"keeping the declaration": err.Error()} + } + } + if applyErr != nil { + report.Failed = map[string]string{"apply": applyErr.Error()} + } + report.Failed = withUnanswered(report.Failed, probe) + // And how each module of the declaration came out, for the have keys written beside this report (novox/hq ADR 0429 + // §4, issue 1132): not on the wire. + report.Modules = moduleFacts(declared, outcome, applyErr) + report.Before = before + // Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire. + report.Unapplied = unapplied(declared, outcome, applyErr) + return report +} + +// factsOf is what this node says of itself: its ports, its profile, how what runs here is, which links face outside, +// what it holds, its filters and, when adopted, its firewall and what is reachable. The facts are an event of the node +// to the controller (novox/hq ADR 0539 §9); declared is the declaration the node was last held to, updated its state, +// outcome what the last apply did (empty when nothing was applied), and held the ids it holds. +func factsOf(ctx context.Context, opts options, declared *declaration.Declaration, updated store.State, + outcome apply.Report, held map[string]bool) link.Report { + report := link.Report{Carried: carriedPorts(updated), Host: runningVersion(), Rollbacks: standingRollbacks(opts.state), Witness: link.WitnessContract, Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))} // What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just @@ -2326,36 +1893,35 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto } report.Applied = append(report.Applied, change.ID) } - // Kept whichever way it went, so a node that is disconnected next minute still knows what it - // was told. Saved after applying rather than before: what is kept is what this node acted on. - var before []string - if signed != nil { - // The modules the declaration kept until now held, so a have key written for one the new declaration no longer - // holds is written removed even by a node-engine that started since (issue 1369). - if kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)); keptErr == nil { - if old, parseErr := declaration.ParseTrusted(kept.Declaration); parseErr == nil { - for _, f := range moduleFacts(old, apply.Report{}, nil) { - before = append(before, f.Module) - } - } - } - if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil { - report.Failed = map[string]string{"keeping the declaration": err.Error()} - } - } - if applyErr != nil { - report.Failed = map[string]string{"apply": applyErr.Error()} - } - report.Failed = withUnanswered(report.Failed, probe) - // And how each module of the declaration came out, for the have keys written beside this report (novox/hq ADR 0429 - // §4, issue 1132): not on the wire. - report.Modules = moduleFacts(declared, outcome, applyErr) - report.Before = before - // Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire. - report.Unapplied = unapplied(declared, outcome, applyErr) return report } +// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs here +// is, and the haves of the modules the declaration it last applied holds. It applies nothing. +func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, say link.Announce) { + known, err := store.Load(opts.state) + if err != nil { + say("cannot say this node's facts: " + err.Error()) + return + } + declared := &declaration.Declaration{} + if raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer); err == nil { + if d, err := declaration.Parse(raw); err == nil { + declared = d + } + } + held := map[string]bool{} + for _, h := range known.Held { + held[h.ID] = true + } + report := factsOf(ctx, opts, declared, known, apply.Report{}, held) + report.Modules = moduleFacts(declared, apply.Report{}, nil) + if report.Modules == nil { + report.Modules = []link.ModuleFact{} + } + session.SayFacts(ctx, report) +} + // waitForEnrolment returns this node's identity, waiting for one if it has none. // // It applies the carried bundle first, if there is one, because that is what a first node does @@ -2488,92 +2054,6 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say os.Exit(0) } -// refuseOlder refuses a declaration from the mesh that is older than the one this node holds. -// -// **By sequence, not by arrival** (novox/hq 04-ISSUES/107). What the host kept is the last thing -// the mesh said, signed; a declaration whose sequence is lower was composed before it, whatever -// order they arrived in — a backlog drained after the node was away, or a broker that split a burst. -// Applying it would make the machine into something the mesh had already moved past, which is the -// incident of issue 104 by another door. -// -// Only when both sides claim an order. A declaration with no sequence is one an older controller -// sent, and one kept with no sequence is one this host received before it understood them; in either -// case there is no order to compare, and refusing on a guess would strand the node the moment the -// controller is older than the host. Equal is the same declaration again, which reconciling is for. -// -// **And by the lease epoch first** (novox/hq to-be 45 §6): a declaration sent under an older epoch than -// the one kept here is a controller that lost its lease, whatever its sequence — the same rule the -// link's deliveries are held to (olderThanKept). Where both claim an epoch it alone decides, because a -// new lease holder's sequence says nothing against the last one's. -func refuseOlder(kept store.Declared, keptErr error, order link.Order) error { - if held, older := olderThanKept(kept, keptErr, order); older { - return fmt.Errorf("this declaration was sent under lease epoch %d, sequence %d, and the one kept "+ - "here under epoch %d, sequence %d: a controller that no longer holds the lease sent it. Refused "+ - "whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence) - } - sequence := order.Sequence - if sequence == 0 || keptErr != nil { - return nil - } - last, err := declaration.ParseTrusted(kept.Declaration) - if err != nil || last.Sequence == 0 || (order.Epoch > 0 && last.Epoch > 0) { - return nil - } - if sequence < last.Sequence { - return fmt.Errorf("this declaration is older than what the mesh last said to this node: it "+ - "is sequence %d, and the one kept here is %d. It arrived late — a backlog, or a broker "+ - "that split a burst — and applying it would make this machine into something the mesh has "+ - "already moved past. Refused whole; nothing was applied", sequence, last.Sequence) - } - return nil -} - -// olderGeneration is the refusal of a declaration composed from an older assignment generation than -// this machine applied, nil when it is not one (novox/hq issue 234). -// -// Only when both claim a generation: a declaration without one is from a controller that claims none, -// and a machine that applied none has nothing to hold it to — refusing on a guess would strand every -// machine the moment the controller is older than the node-engine. Equal is not older: a push of the -// same assignments is the ordinary case. A put-back carries the generation of what it puts back and -// says so; a gate putting a machine back is the one sender that may say something older. -func olderGeneration(known store.State, d *declaration.Declaration) *link.GenerationRefusal { - if d.PutBack || d.Generation <= 0 || known.Generation <= 0 || d.Generation >= known.Generation { - return nil - } - return &link.GenerationRefusal{Generation: d.Generation, Applied: known.Generation} -} - -// olderGenerationWords is a generation refusal as the report and a person read it. -func olderGenerationWords(r link.GenerationRefusal) string { - return fmt.Sprintf("this declaration was composed from assignment generation %d, and this node applied "+ - "generation %d: it was composed from a view of the assignments the mesh has moved past, and applying it "+ - "would undeclare what the mesh still assigns here (novox/hq issue 234). Refused whole; nothing was "+ - "applied", r.Generation, r.Applied) -} - -// appliedGeneration is the highest generation applied once this declaration is: raised by it, never -// lowered — a put-back of an older generation leaves it where it was (novox/hq issue 234). -func appliedGeneration(known store.State, d *declaration.Declaration) int64 { - return max(known.Generation, d.Generation) -} - -// olderThanKept says whether a declaration of this order is older than the one this node kept — the -// last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept -// declaration that cannot be read claims no order: what it is compared with is unknown, and refusing -// on a guess would hold the machine off everything the mesh sends until a person looked. That kept -// file is read, and refused by name, by the next reconcile. -func olderThanKept(kept store.Declared, keptErr error, order link.Order) (link.Order, bool) { - if keptErr != nil || order.Epoch <= 0 { - return link.Order{}, false - } - last, err := declaration.ParseTrusted(kept.Declaration) - if err != nil { - return link.Order{}, false - } - held := link.Order{Epoch: last.Epoch, Sequence: last.Sequence} - return held, order.Older(held) -} - // profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a // report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161). func profileAsReported(detected profile.Profile) map[string]any { diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 1aaf8bf..f33c638 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "crypto/ed25519" "encoding/json" "errors" "os" @@ -14,8 +13,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/bundle" - "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" @@ -151,77 +148,6 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) { } } -// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds -// or its firewall changed — which is how a predecessor still writing is caught, without a report -// every five minutes saying nothing new. -func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) { - w := &adoptionWatch{} - held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}} - if !w.changed(held) { - t.Fatal("the first report of a hold was not said") - } - again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}} - if w.changed(again) { - t.Error("the same holds in another order were said again") - } - rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}} - if !w.changed(rewritten) { - t.Error("a held file rewritten by something else was not said") - } - if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) { - t.Error("a changed firewall was not said") - } - // What filters the machine is part of it (novox/hq ADR 0168): a predecessor's chain removed by - // hand, or the found firewall enabled again, is said without being asked. - filtered := link.Report{Firewall: "none", Held: rewritten.Held, - Filters: []link.Filter{{Where: "chain HAL-MESH-ONLY (iptables-legacy)", Owner: "other", Refuses: "-j DROP"}}} - if !w.changed(filtered) { - t.Error("a filter appearing was not said") - } - if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) { - t.Error("a filter removed was not said") - } - if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held, FoundFirewall: &link.FoundFirewall{Kind: "ufw", Active: true}}) { - t.Error("the found firewall coming back was not said") - } - if !worthSaying(link.Report{Filters: filtered.Filters}) { - t.Error("a report carrying only what filters the machine is not worth saying") - } -} - -// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue -// tells the watch every report the broker took (link.Queue.Heard). -func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) { - w := &adoptionWatch{} - report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}} - heard := w.said - heard(report) - if w.changed(report) { - t.Error("a reconcile repeated what the mesh had just heard") - } -} - -func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) { - // The controller previews a flip from what the node last said is reachable; a port that opened - // since must reach it without waiting for the next delivery (novox/hq ADR 0100). - w := &adoptionWatch{} - before := link.Report{Firewall: "ufw", Reachable: []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} - if !w.changed(before) { - t.Fatal("the first report was not said") - } - reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} - if w.changed(reordered) { - t.Error("the same reachable set was said again") - } - opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable, - link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})} - if !w.changed(opened) { - t.Error("a newly published port was not said") - } -} - // Defends the node's own record: the link and the reconcile loop both apply, and each reads the // state, acts, and writes it back — so they must not run at the same time, or the last save loses // what the other recorded. @@ -270,24 +196,6 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { } } -// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued -// and lost — the link down when the reconcile spoke — it must be said again. -func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) { - w := &adoptionWatch{} - held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}} - if !w.differs(held) { - t.Fatal("the first report of a change was not new") - } - // The link was down: nothing published it, so nothing says it was said. - if !w.differs(held) { - t.Error("a change that never reached the mesh was counted as said") - } - w.said(held) - if w.differs(held) { - t.Error("a change the mesh was told was said again") - } -} - // Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at // the point of application, whichever command delivered it, naming both — an adopted control-node // once applied its converged genesis bundle and closed itself for forty-five minutes. @@ -468,125 +376,6 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) { } } -// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to -// that — never to the bundle the host carries, which genesis consumed. -func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) { - was := builtFor - builtFor = "arch" - t.Cleanup(func() { builtFor = was }) - opts := stateWithMode(t, store.ModeAdopted) - - controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`) - if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said, - Signature: ed25519.Sign(controllerPrivate, said)}); err != nil { - t.Fatal(err) - } - - // Told, and unable to prove by whom: refused, and the bundle is not applied in its place. - _, _, _, err = reconcileSource(opts) - if err == nil || !strings.Contains(err.Error(), "not applied in its place") { - t.Errorf("a node that cannot prove what it was told fell back to something: %v", err) - } - - mine, err := identity.Generate("workstation") - if err != nil { - t.Fatal(err) - } - mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0", - Signer: controllerPublic, Password: "issued"} - if err := identity.Save(identity.Path(opts.state), mine); err != nil { - t.Fatal(err) - } - d, raw, from, err := reconcileSource(opts) - if err != nil { - t.Fatal(err) - } - if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil { - t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw)) - } - - // And before the mesh has said anything: the bundle, as it always was. A test binary carries - // only the placeholder, and asking for it is what proves the path. - if err := os.Remove(store.DeclaredPath(opts.state)); err != nil { - t.Fatal(err) - } - _, _, _, err = reconcileSource(opts) - if !errors.Is(err, bundle.ErrEmpty) { - t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err) - } -} - -// **A machine says which links face outside without being asked.** -// -// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only -// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say -// could speak only when a declaration arrived — and a declaration cannot be composed until it has -// spoken. A machine waiting for a push that is waiting for the machine. -func TestANewOutwardLinkIsSaidUnasked(t *testing.T) { - w := &adoptionWatch{} - first := link.Report{Firewall: "none"} - if !w.differs(first) { - t.Fatal("the first report should differ from nothing") - } - w.said(first) - - // Only the links changed, and nothing about adoption. - learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}} - if !w.differs(learnt) { - t.Fatal("a machine that has just learnt which links face outside would never say so, " + - "and could then never be sent a filter") - } - w.said(learnt) - if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) { - t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile") - } - - // And a link that changes — a laptop moving from a cable to a radio — is said too, because the - // filter is written around the old one until it is. - if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) { - t.Fatal("a changed outward link is not said, so the filter stays written around the old one") - } -} - -// **A converged machine can say which links face outside, unasked.** -// -// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an -// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so -// it could speak only in reply to a declaration, and the mesh composes no declaration for a node -// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines -// sat silent while the control plane refused to send them a filter. -func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) { - // A converged node's reconcile: nothing held, no found firewall, and the links it can see. - converged := link.Report{Outward: []string{"eth0"}} - if !worthSaying(converged) { - t.Fatal("a converged machine cannot say which links face outside, so it can never be " + - "sent a filter — a machine waiting for a push that is waiting for the machine") - } - - // An adopted node's reasons still hold, because that is how a predecessor still writing is caught. - if !worthSaying(link.Report{Firewall: "ufw"}) { - t.Fatal("an adopted machine no longer says which firewall it found") - } - if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) { - t.Fatal("an adopted machine no longer says what it holds") - } - - // And a reconcile with nothing to say stays silent, or every machine speaks every five minutes - // about nothing. - if worthSaying(link.Report{}) { - t.Fatal("a reconcile with nothing to say speaks anyway") - } - - // A refused report says nothing about the machine; the refusal is for the console. - if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) { - t.Fatal("a refused report is offered as news about the machine") - } -} - // **A host running as a service says what its apply did.** // // The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path @@ -611,63 +400,3 @@ func TestTheApplysLogIsNeverNil(t *testing.T) { t.Fatalf("the apply's detail did not reach the caller's announce: %v", said) } } - -// Defends novox/hq issues 257 and 261: a reconcile due while the link applies a newer declaration -// applies that newer one once its turn comes — never the one kept when its timer fired. Read before -// waiting, it gave back a module assigned a second earlier, and reported a declaration the mesh no -// longer recorded as sent. -func TestAReconcileAppliesWhatWasKeptWhenItsTurnComes(t *testing.T) { - built, err := system.For("arch") - if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { - t.Skip("this machine is not one these tests can apply on") - } - was := builtFor - builtFor = "arch" - t.Cleanup(func() { builtFor = was }) - dir := t.TempDir() - opts := options{state: filepath.Join(dir, "state.json")} - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - target := filepath.Join(dir, "a.conf") - kept := func(content string) store.Declared { - body := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + target + - `","content":"` + content + `\n"}]}`) - return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)} - } - if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("older")); err != nil { - t.Fatal(err) - } - - // The link is applying: the reconcile's timer fires and it waits its turn. - applying.Lock() - done := make(chan error, 1) - go func() { - _, err := reapplyKept(context.Background(), opts, public, nil, nil) - done <- err - }() - time.Sleep(50 * time.Millisecond) - // The link's apply ends by keeping the newer declaration, and lets go. - if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("newer")); err != nil { - applying.Unlock() - t.Fatal(err) - } - applying.Unlock() - - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(10 * time.Second): - t.Fatal("the reconcile never ran once the node was free") - } - got, err := os.ReadFile(target) - if err != nil { - t.Fatal(err) - } - if string(got) != "newer\n" { - t.Errorf("the reconcile applied %q, what was kept before the newer declaration", got) - } -} diff --git a/cmd/mesh-host/meshcli.go b/cmd/mesh-host/meshcli.go index c145a96..8960c27 100644 --- a/cmd/mesh-host/meshcli.go +++ b/cmd/mesh-host/meshcli.go @@ -11,14 +11,14 @@ import ( // serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account // from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served // is said once, and this engine goes on applying; mesh-cli then says the engine does not answer. -func serveMeshCLI(ctx context.Context, queue *link.Queue, say func(string)) { +func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) { if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil { say("mesh-cli is not answered on this machine: " + err.Error()) } } // forwardOver asks the controller over the queue's link, and follows the line to what came of it. -func forwardOver(queue *link.Queue) meshcli.Forward { +func forwardOver(queue *link.Session) meshcli.Forward { return func(ctx context.Context, asked meshcli.Asked) cli.Answer { return meshcli.AskTheController(ctx, queue.AskController, asked) } diff --git a/cmd/mesh-host/movesecret.go b/cmd/mesh-host/movesecret.go index 7bbc2ec..da6c23c 100644 --- a/cmd/mesh-host/movesecret.go +++ b/cmd/mesh-host/movesecret.go @@ -27,7 +27,7 @@ type vaultAsk func(verb string, args any) (json.RawMessage, error) // moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over // the link open now. -func moveSecretFrom(statePath string, queue *link.Queue) link.MoveSecretFunc { +func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc { return func(m link.MoveSecret) (link.MovedSecret, error) { key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath)) if err != nil { diff --git a/cmd/mesh-host/nodeclare_test.go b/cmd/mesh-host/nodeclare_test.go new file mode 100644 index 0000000..430fec3 --- /dev/null +++ b/cmd/mesh-host/nodeclare_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// R6-A AC8: nothing in the node-engine receives, orders or reconciles a whole declaration (novox/hq ADR 0539 §9, ADR +// 0429 §8): no package subscribes to a node's declare subject or names the deleted symbols. +func TestNoPackageHearsOrOrdersAWholeDeclaration(t *testing.T) { + deleted := []string{ + "DeclareSubject", "mesh.node.\" + node + \".declare", "ReconcileDue", "ReconcileEvery", "olderThanKept", + "refuseOlder", "olderGeneration", "appliedGeneration", "reconcileKept", "reapplyKept", "adoptionWatch", + "unsaidFile", "numbersFile", "func gather(", "func handleBody(", "Supersedes(", "ReportSequence", + "RefusedOlder", "ReadsGeneration", "ReadsModules", "GenerationRefusal", "LeftOutModuleOf", + } + root := filepath.Join("..", "..") + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for _, word := range deleted { + if strings.Contains(string(body), word) { + t.Errorf("%s still names %q", path, word) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/mesh-host/order_test.go b/cmd/mesh-host/order_test.go deleted file mode 100644 index 1cf9d90..0000000 --- a/cmd/mesh-host/order_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq -// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said, -// "older" could not. - -func keptWith(t *testing.T, sequence int64) store.Declared { - t.Helper() - return keptAt(t, link.Order{Sequence: sequence}) -} - -func keptAt(t *testing.T, order link.Order) store.Declared { - t.Helper() - body, err := json.Marshal(map[string]any{ - "declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence, - "epoch": order.Epoch, - }) - if err != nil { - t.Fatal(err) - } - return store.Declared{Declaration: body, Signature: []byte("x")} -} - -func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) { - err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5}) - if err == nil { - t.Fatal("sequence 5 was accepted over a kept 7") - } - if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") { - t.Fatalf("the refusal does not say what it is: %v", err) - } -} - -func TestANewerOrEqualDeclarationIsNot(t *testing.T) { - if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil { - t.Fatalf("sequence 8 was refused over a kept 7: %v", err) - } - // Equal is the same declaration again, which reconciling is for. - if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil { - t.Fatalf("the same sequence was refused: %v", err) - } -} - -func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) { - // An older controller sends none; a host that received before it understood them kept none. - // Refusing on a guess would strand a node the moment the controller is older than the host. - if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil { - t.Fatalf("a declaration claiming no order was refused: %v", err) - } - if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil { - t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err) - } - if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil { - t.Fatalf("a first declaration was refused: %v", err) - } -} - -// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost -// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence. -func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) { - kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3}) - err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12}) - if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") { - t.Fatalf("an older epoch was not refused by name: %v", err) - } - if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil { - t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err) - } - if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil { - t.Fatal("a lower sequence under the same epoch was accepted") - } - // A controller with no lease — older, or rolled back to a build from before it — is today's - // behaviour: no epoch compared. - if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil { - t.Fatalf("a declaration with no epoch was refused for one: %v", err) - } - if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older || - held != (link.Order{Epoch: 57, Sequence: 3}) { - t.Fatalf("olderThanKept said %v, holding %+v", older, held) - } - // What is kept cannot be read: no order to compare, so nothing is refused on a guess. - if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil, - link.Order{Epoch: 41, Sequence: 1}); older { - t.Fatal("refused against a kept declaration nobody can read") - } -} diff --git a/cmd/mesh-host/probe_test.go b/cmd/mesh-host/probe_test.go index 6433fd2..357aa1e 100644 --- a/cmd/mesh-host/probe_test.go +++ b/cmd/mesh-host/probe_test.go @@ -9,7 +9,6 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/declaration" - "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/system" ) @@ -127,83 +126,3 @@ func TestAPackageManagerThatIsNotThereRefusesWhole(t *testing.T) { t.Error("the refused declaration touched the machine") } } - -// Defends the review of novox/hq issue 233: **a re-apply whose result changed is said unasked**. A machine -// reconciles every five minutes and speaks only when its account is news; how the apply went was not part of -// it, so a package manager that stopped answering between deliveries was never said, and one that answered -// again never cleared. Now a failure is said once, the same failure again is not, and a clean re-apply after it -// is — which is what clears the mesh's condition. -func TestAReapplyWhoseResultChangedIsSaidUnasked(t *testing.T) { - dir := t.TempDir() - conf := filepath.Join(dir, "elsewhere", "pacman.conf") // not declared: the apply cannot make it answer - pacmanAnsweringOnce(t, conf) - opts := options{state: filepath.Join(dir, "state.json")} - raw := []byte(`{"declaration":1,"resources":[` + - `{"id":"motd","type":"file","path":"` + filepath.Join(dir, "motd") + `","content":"hello\n"},` + - `{"id":"pacman.base","type":"package","package":"base"}]}`) - w := &adoptionWatch{} - - // Clean once, as it was before: base installed while pacman answered. - if err := os.MkdirAll(filepath.Dir(conf), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil { - t.Fatal(err) - } - clean := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) - if faulty(clean) { - t.Fatalf("the first apply was not clean: %+v", clean) - } - w.said(clean) - if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) { - t.Error("a clean re-apply after a clean one was said, so every machine speaks every five minutes") - } - - // The configuration goes: the re-apply fails, and that is news once. - if err := os.Remove(conf); err != nil { - t.Fatal(err) - } - failed := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) - if !faulty(failed) { - t.Fatalf("the re-apply did not fail: %+v", failed) - } - if !w.news(failed) { - t.Fatal("a re-apply that failed after a clean one was not said, so the mesh never hears the machine fails") - } - w.said(failed) - if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) { - t.Error("the same failure was said again on the next re-apply") - } - - // It comes back: the clean re-apply is news, and is what clears the mesh's condition. - if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil { - t.Fatal(err) - } - again := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) - if faulty(again) { - t.Fatalf("the re-apply did not come back clean: %+v", again) - } - if !w.news(again) { - t.Fatal("a clean re-apply after a failed one was not said, so the mesh's condition never clears") - } -} - -// A refusal is news too, once, and a reconcile with nothing to say after a clean one stays silent. -func TestARefusalIsSaidOnceAndNothingIsNotSaid(t *testing.T) { - w := &adoptionWatch{} - w.said(link.Report{}) - if w.news(link.Report{}) { - t.Error("nothing was said again") - } - refused := link.Report{Refused: "pacman does not answer here"} - if !w.news(refused) { - t.Fatal("a refusal was not said") - } - w.said(refused) - if w.news(link.Report{Refused: "pacman does not answer here, at another time"}) { - t.Error("the same refusal in other words was said again") - } - if !w.news(link.Report{}) { - t.Error("a clean apply after a refusal was not said") - } -} diff --git a/cmd/mesh-host/unsaid_test.go b/cmd/mesh-host/unsaid_test.go deleted file mode 100644 index 4c63317..0000000 --- a/cmd/mesh-host/unsaid_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// The last apply's report, kept beside the state until the mesh takes it (novox/hq issue 264). - -// What a host that died between applying and reporting kept is read back by the next one exactly — -// the digest and the outcome, never a new account of the machine. -func TestTheKeptReportSurvivesTheHost(t *testing.T) { - state := filepath.Join(t.TempDir(), "state.json") - made := link.Report{Declared: "sha256:d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"}, - Host: "v2"} - if err := unsaidBeside(state).Keep(made); err != nil { - t.Fatal(err) - } - - kept, ok, err := unsaidBeside(state).Pending() - if err != nil || !ok { - t.Fatalf("nothing kept: %v", err) - } - if !reflect.DeepEqual(kept, made) { - t.Fatalf("kept %+v, made %+v", kept, made) - } - - // Taken about another declaration, it stays; taken about this one, it goes. - if err := unsaidBeside(state).Said("sha256:other"); err != nil { - t.Fatal(err) - } - if _, ok, _ := unsaidBeside(state).Pending(); !ok { - t.Fatal("a report about another declaration cleared this one") - } - if err := unsaidBeside(state).Said("sha256:d1"); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(store.UnsaidPath(state)); !os.IsNotExist(err) { - t.Fatalf("the report was taken and is still kept: %v", err) - } -} - -// A node that never applied anything has nothing to say again; and an apply refused before its -// declaration was read clears an older report, which would otherwise be re-said after it. -func TestNothingKeptWhenNothingWasApplied(t *testing.T) { - state := filepath.Join(t.TempDir(), "state.json") - if _, ok, err := unsaidBeside(state).Pending(); ok || err != nil { - t.Fatalf("a node that applied nothing has a report to say: %v %v", ok, err) - } - if err := unsaidBeside(state).Keep(link.Report{Declared: "sha256:d1"}); err != nil { - t.Fatal(err) - } - if err := unsaidBeside(state).Keep(link.Report{Refused: "forged"}); err != nil { - t.Fatal(err) - } - if _, ok, _ := unsaidBeside(state).Pending(); ok { - t.Fatal("an older report is kept after a later declaration was refused") - } -} diff --git a/cmd/mesh-host/vaultbox_nats_test.go b/cmd/mesh-host/vaultbox_nats_test.go index 719040c..30e0fd5 100644 --- a/cmd/mesh-host/vaultbox_nats_test.go +++ b/cmd/mesh-host/vaultbox_nats_test.go @@ -22,7 +22,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/identity" - "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) @@ -61,14 +60,14 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear( } _ = tap.Flush() - // The engine hears its declaration on its own subject. - const node = "vaultbox" + // The engine hears the composition on a subject of the test (the node agent reads it from its bucket). + const subject = "mesh.test.vaultbox.composition" engine, err := nats.Connect(url) if err != nil { t.Fatal(err) } defer engine.Close() - heard, err := engine.SubscribeSync(link.DeclareSubject(node)) + heard, err := engine.SubscribeSync(subject) if err != nil { t.Fatal(err) } @@ -92,7 +91,7 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear( t.Fatal(err) } defer controller.Close() - if err := controller.Publish(link.DeclareSubject(node), body); err != nil { + if err := controller.Publish(subject, body); err != nil { t.Fatal(err) } _ = controller.Flush() @@ -123,7 +122,7 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear( mu.Lock() all := strings.Join(crossed, "\n") mu.Unlock() - if !strings.Contains(all, link.DeclareSubject(node)) || !strings.Contains(all, "vault-sealed") { + if !strings.Contains(all, subject) || !strings.Contains(all, "vault-sealed") { t.Fatalf("the listener heard nothing of the declaration:\n%s", all) } for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} { diff --git a/cmd/mesh-host/witnessing.go b/cmd/mesh-host/witnessing.go index bf7bffb..d4aa7a0 100644 --- a/cmd/mesh-host/witnessing.go +++ b/cmd/mesh-host/witnessing.go @@ -41,8 +41,9 @@ func rolledBackHosts(statePath string) []string { return upgrade.RolledBackVersions(verdicts) } -// proving is this host waiting for the mesh to take a report it made under its own build — the -// evidence its launcher's trial waits for — and acting on it once. +// proving is this host waiting to write a have key under its own build — the evidence its launcher's trial waits for +// (novox/hq ADR 0539 §9, ADR 0227 §8: the component replaced is never the only witness of its successor) — and acting +// on it once. type proving struct { statePath string version string @@ -56,16 +57,14 @@ func unproved(statePath, version string) bool { return err != nil || known != version } -// heard is told every account of the machine the mesh has taken. The first one about a declaration, -// made by this build, proves it. -func (p *proving) heard(r link.Report) { - if r.Declared == "" || r.Refused != "" || r.Host != p.version { - return - } +// haveWritten is told when this build has written a have key, or found the bucket already holding what it would write. +// The first proves it. **A declaration's report does not**: a node-engine that receives none is not on trial for it, and +// one that cannot write its haves is what the launcher keeps rolling back. +func (p *proving) haveWritten() { p.once.Do(func() { retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version) if err != nil { - p.say(fmt.Sprintf("this host reported under its own build %s, and proving it was not complete: %v", p.version, err)) + p.say(fmt.Sprintf("this host wrote its first have under its own build %s, and proving it was not complete: %v", p.version, err)) } if len(retired) > 0 { p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", "))) @@ -146,7 +145,7 @@ func launcherReplaced(pid int, proc, path string) bool { } // watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8). -func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue, say func(string)) { +func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) { host, _ := os.Hostname() w := &witness.Watcher{ Root: apply.DaemonRoot(), Node: node, Host: host, @@ -159,7 +158,7 @@ func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue }, // Said now, not at the next report: the verdict is on every report from here, and the mesh // is asked for one at once. - Concluded: func(link.Rollback) { queue.ReportAsked() }, + Concluded: func(link.Rollback) { queue.FactsAsked(ctx) }, Say: say, } w.Watch(ctx) diff --git a/cmd/mesh-host/witnessing_test.go b/cmd/mesh-host/witnessing_test.go index 13ee112..3e304b6 100644 --- a/cmd/mesh-host/witnessing_test.go +++ b/cmd/mesh-host/witnessing_test.go @@ -13,9 +13,9 @@ import ( "github.com/novox/mesh-host/internal/upgrade" ) -// The first account the mesh takes from this build, about a declaration, proves it to the launcher; -// nothing else does — a refusal, a report about no declaration, another build's report (to-be 45 §8). -func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) { +// The first have this build writes proves it to the launcher; nothing else does — not a report about a declaration, not +// a facts event, not the haves of another build (novox/hq ADR 0539 §9, ADR 0227 §8). R6-A AC7. +func TestOnlyAHaveUnderThisBuildProvesIt(t *testing.T) { state := filepath.Join(t.TempDir(), "state.json") t.Setenv("MESH_HOST_LIBEXEC", t.TempDir()) // Never the machine's own engine placed by hand: proving renews it (issue 356). @@ -27,22 +27,20 @@ func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) { var said []string p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }} - for _, r := range []link.Report{ - {Host: "2.0", Refused: "no"}, - {Host: "2.0"}, - {Host: "1.0", Declared: "abc"}, - } { - p.heard(r) - if !unproved(state, "2.0") { - t.Fatalf("%+v proved this build", r) - } - if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" { - t.Fatalf("%+v renewed the engine placed by hand", r) - } + // Facts said, a link opened, a declaration of the old kind reported: none of it is a have. + if !unproved(state, "2.0") { + t.Fatal("a build that wrote no have is proved") } - p.heard(link.Report{Host: "2.0", Declared: "abc"}) + if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" { + t.Fatal("the engine placed by hand was renewed before any have") + } + + // The session tells the proof when it has a have key saying what it would write. + session := &link.Session{HaveWritten: p.haveWritten} + _ = session + p.haveWritten() if unproved(state, "2.0") { - t.Fatalf("an account of a declaration under this build did not prove it (%v)", said) + t.Fatalf("the first have under this build did not prove it (%v)", said) } if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" { t.Fatalf("known-good is %q", got) @@ -78,12 +76,6 @@ func TestAStandingRollbackIsSaidOnEveryReport(t *testing.T) { got[0].Outcome != link.RolledBack { t.Fatalf("what the report says is %+v", got) } - if !worthSaying(link.Report{Rollbacks: got}) { - t.Fatal("a reconcile with a rollback standing does not say it") - } - if adoptionFingerprint(link.Report{Rollbacks: got}) == adoptionFingerprint(link.Report{}) { - t.Fatal("a rollback reached or ended is not news to the reconcile") - } } // The launcher running this host is known to have been replaced only when it certainly was: a shell diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 83faeeb..03c57b6 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -21,7 +21,6 @@ import ( "os" "os/exec" "path/filepath" - "slices" "sort" "strconv" "strings" @@ -561,18 +560,6 @@ func applyScoped( protecting = append(protecting, orphan) continue } - // **A module the mesh left out is not a module the mesh removed** (novox/hq ADR 0163, rule - // 6): its resources are absent because a setting stored for it cannot compose, and the - // mesh said so by name. What the host wrote for it stays as it is, recorded, until the - // module is declared again or unassigned. - if module, left := d.LeftOutModuleOf(orphan.ID); left { - report.Outcomes = append(report.Outcomes, Outcome{ - ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, - Action: "unchanged", Detail: "kept: " + module + " was left out of this declaration by the mesh, not removed", - }) - log(fmt.Sprintf(" kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module)) - continue - } if by, replaced := replacedBy[orphan.ID]; replaced { handover[by] = append(handover[by], orphan) continue @@ -616,11 +603,6 @@ func applyScoped( if declared[h.ID] || (scope != nil && h.Module != scope.Module) { continue } - if slices.Contains(d.LeftOut, h.Module) { - // Left out, not unassigned (ADR 0163, rule 6): still held for the module, as the - // mesh asked. - continue - } known.Release(h.ID) report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target, Action: "forgotten", Detail: "no longer declared; left as found"}) diff --git a/internal/apply/left_out_test.go b/internal/apply/containers_kept_test.go similarity index 69% rename from internal/apply/left_out_test.go rename to internal/apply/containers_kept_test.go index a7b11a5..3bc8684 100644 --- a/internal/apply/left_out_test.go +++ b/internal/apply/containers_kept_test.go @@ -75,69 +75,6 @@ func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) { var specOfLast string -// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163, -// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held. -// A module simply absent is removed as it always was. -func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) { - var removed []string - gone := map[string]bool{} - run := func(_ context.Context, name string, args ...string) (string, error) { - args = oldShape(name, args) - if name != "docker" { - return "", nil - } - switch args[0] { - case "info": - return "29.0.0\n", nil - case "rm": - removed = append(removed, args[len(args)-1]) - gone[args[len(args)-1]] = true - case "container": - if gone[args[len(args)-1]] { - return "", errors.New("no such container") - } - return "true\tspec", nil - } - return "", nil - } - known := store.State{ - Resources: []store.Applied{ - {ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared}, - {ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared}, - }, - Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}}, - } - d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[ - {"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"} - ]}`) - report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) - if err != nil { - t.Fatal(err) - } - if len(removed) != 1 || removed[0] != "old" { - t.Fatalf("removed %v; only the module that is absent goes", removed) - } - if _, kept := state.At("container", "web"); !kept { - t.Fatal("the left-out module's record was forgotten") - } - if _, held := state.HeldAt("web.page"); !held { - t.Fatal("the left-out module's hold was released") - } - said := false - for _, o := range report.Outcomes { - if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") { - said = true - } - if o.ID == "web.server" && o.Action != "unchanged" { - t.Errorf("the left-out module's container was %s", o.Action) - } - } - if !said { - t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes) - } -} - -// A container's capabilities reach the runtime and are part of its spec (novox/hq ADR 0170). func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) { var ran []string run := func(_ context.Context, name string, args ...string) (string, error) { diff --git a/internal/link/asked_test.go b/internal/link/asked_test.go deleted file mode 100644 index 8211c4b..0000000 --- a/internal/link/asked_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package link - -import ( - "context" - "strings" - "testing" - "time" -) - -// The `report` verb (novox/hq to-be 45 §6), what healer H1 asks a machine whose send went unreported: -// the node says again what it last applied, as an ordinary report through the one queue. - -// **Asked, the reconcile's account is said even when it is not news** — the mesh asked because it does -// not know, and silence would be the very fault it is asking about. -func TestAskedToReportSaysWhatWasLastAppliedEvenWhenNotNews(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - var lines []string - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) - q.Say = func(s string) { lines = append(lines, s) } - q.Reconcile = func(context.Context) (Report, bool) { - return Report{Declared: "d-kept", Order: Order{Epoch: 7, Sequence: 12}, Applied: []string{"a"}}, true - } - q.News = func(Report) bool { return false } - q.attach(context.Background(), l) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - - q.ReconcileDue() // an ordinary reconcile: not news, not said - time.Sleep(50 * time.Millisecond) - if n := len(l.said()); n != 0 { - t.Fatalf("an ordinary reconcile that is not news said %d report(s)", n) - } - q.ReportAsked() - eventually(t, "asked, the node said nothing", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - r := l.said()[0] - if r.Declared != "d-kept" || r.Sequence != 12 || r.Epoch != 7 || r.ReportSequence == 0 || r.Node != m.Node { - t.Fatalf("the account said is not the one kept, in order: %+v", r) - } - if !strings.Contains(strings.Join(lines, "\n"), "asked by the mesh") { - t.Errorf("being asked is not said in the journal: %q", lines) - } -} - -// **A delivery waiting answers the question better**: it is applied, once, and its report is the answer. -func TestAskedWhileADeliveryWaitsIsThatDeliverysReport(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - reconciled := 0 - q.Reconcile = func(context.Context) (Report, bool) { reconciled++; return Report{}, true } - q.attach(context.Background(), l) - q.Deliver(ordered(t, key, "new", 7, 13)) - q.ReportAsked() - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - time.Sleep(50 * time.Millisecond) - stop() - <-worker - if reconciled != 0 || len(l.said()) != 1 || l.said()[0].Sequence != 13 { - t.Fatalf("reconciled %d, said %+v", reconciled, l.said()) - } -} - -// askingLink is a quiet link the mesh can ask. -type askingLink struct { - *quietLink - asks chan struct{} -} - -func (l askingLink) AskedToReport() <-chan struct{} { return l.asks } - -// **The link hands the question to the queue**: asked on the link, the account is said on it. -func TestALinkThatIsAskedHandsItToTheQueue(t *testing.T) { - m, _ := aMember(t) - l := askingLink{quietLink: newQuietLink(), asks: make(chan struct{}, 1)} - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) - q.Reconcile = func(context.Context) (Report, bool) { return Report{Declared: "d-kept"}, true } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - worker := runWorker(ctx, q) - served := make(chan error, 1) - go func() { served <- serve(ctx, l, m, q, nil, time.Second) }() - l.asks <- struct{}{} - eventually(t, "asked on the link, nothing was said", func() bool { - for _, r := range l.said() { - if r.Declared == "d-kept" { - return true - } - } - return false - }) - stop() - <-served - <-worker -} - -// **Against a real server**: the question on the node's own subject reaches it, and another's does not. -func TestNatsTheMeshAsksANodeToReport(t *testing.T) { - conn, js := aBus(t) - l := &natsLink{conn: conn, js: js, node: "asked"} - l.hearAsks() - defer l.Close() - if err := conn.Publish(AskReportSubject("another"), []byte(`{}`)); err != nil { - t.Fatal(err) - } - if err := conn.Publish(AskReportSubject("asked"), []byte(`{"by":"the controller's healer H1"}`)); err != nil { - t.Fatal(err) - } - if err := conn.Flush(); err != nil { - t.Fatal(err) - } - select { - case <-l.AskedToReport(): - case <-time.After(5 * time.Second): - t.Fatal("the question on the node's own subject never reached it") - } - select { - case <-l.AskedToReport(): - t.Fatal("asked twice: another node's question reached this one") - case <-time.After(200 * time.Millisecond): - } - if AskReportSubject("asked") != "mesh.node.asked.ask.report" { - t.Fatalf("the subject %q is not the one the controller asks on and grants", AskReportSubject("asked")) - } -} diff --git a/internal/link/asktool_nats_test.go b/internal/link/asktool_nats_test.go index ca34dbd..c1a4bcc 100644 --- a/internal/link/asktool_nats_test.go +++ b/internal/link/asktool_nats_test.go @@ -25,7 +25,7 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) { t.Fatal(err) } - q := &Queue{Membership: Membership{Node: node}} + q := &Session{Membership: Membership{Node: node}} q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node}) ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index eaa284f..3442a33 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -184,7 +184,7 @@ func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h } func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { m, _ := aMember(t) l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)} - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) + q := &Session{Membership: m, Timeout: time.Second} q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil } ctx, stop := context.WithCancel(context.Background()) served := make(chan error, 1) diff --git a/internal/link/have.go b/internal/link/have.go index 6428a36..702ca78 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -97,7 +97,7 @@ func nodeFacts(r Report) delivery.HaveNode { } // noteApply takes what an apply's report says of the modules and the node. -func (q *Queue) noteApply(r Report) bool { +func (q *Session) noteApply(r Report) bool { if r.Modules == nil { return false } @@ -146,7 +146,7 @@ func (q *Queue) noteApply(r Report) bool { } // noteHealth takes a look's health, and says whether any module's changed. -func (q *Queue) noteHealth(h Health) bool { +func (q *Session) noteHealth(h Health) bool { s := &q.haves s.mu.Lock() defer s.mu.Unlock() @@ -170,7 +170,7 @@ func (q *Queue) noteHealth(h Health) bool { } // sayHave is a line in the node-engine's log, if it has one. -func (q *Queue) sayHave(s string) { +func (q *Session) sayHave(s string) { if q.Say != nil { q.Say(s) } @@ -180,7 +180,7 @@ func (q *Queue) sayHave(s string) { // §4, issue 1132): each module's have, and the node's own facts. It reads the key first and writes nothing when the value // is the same, otherwise writes over the revision it read, and creates the key when there is none. A write the bucket // refuses is said and left: the next apply or look writes again, and the report was sent before this ran. -func (q *Queue) writeHaves(ctx context.Context, bus Bus) { +func (q *Session) writeHaves(ctx context.Context, bus Bus) { hb, ok := bus.(HaveBus) if !ok || bus == nil { return @@ -261,7 +261,7 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) { // writeRemoved writes, once, each module the applied declaration no longer holds and this node-engine wrote a have key // for, with the phase removed over the revision it reads (issue 1369). The key is never deleted, and a key that is not // in the bucket is not created. False when the pass should stop. The caller holds the lock of the haves. -func (q *Queue) writeRemoved(ctx context.Context, b HaveBucket, node string) bool { +func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) bool { s := &q.haves gone := make([]string, 0, len(s.removed)) for module := range s.removed { @@ -311,7 +311,7 @@ func (q *Queue) writeRemoved(ctx context.Context, b HaveBucket, node string) boo // putHave reads a key and writes it unless same says it already holds the value. False when the pass should stop: the // bucket failed, the error is said once, and the rest wait for the next pass. -func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool { +func (q *Session) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool { if ctx.Err() != nil { q.sayHave(fmt.Sprintf("the time for writing the have bucket ran out before %s; the next apply or look writes it", key)) return false @@ -324,11 +324,22 @@ func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []b q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err)) return false case same(current): + q.haveWritten() return true } if err := b.Set(ctx, key, value, revision); err != nil { q.sayHave(fmt.Sprintf("could not write %s in the have bucket; the next apply or look writes it again: %v", key, err)) return false } + q.haveWritten() return true } + +// haveWritten tells the caller this node-engine has a have key in the bucket saying what it says now: written by it, or +// already holding the same. It is what proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9): the first +// have under its own build, not a declaration's report. +func (q *Session) haveWritten() { + if q.HaveWritten != nil { + q.HaveWritten() + } +} diff --git a/internal/link/have_nats_integration_test.go b/internal/link/have_nats_integration_test.go index 1ff1f03..cc7a6ff 100644 --- a/internal/link/have_nats_integration_test.go +++ b/internal/link/have_nats_integration_test.go @@ -4,7 +4,6 @@ package link import ( "context" - "encoding/json" "sync/atomic" "testing" "time" @@ -32,26 +31,20 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { _ = kv.Purge(node + "." + name) } - m, key := aMember(t) + m, _ := aMember(t) m.Node = node var failB atomic.Bool - var applied atomic.Int32 failB.Store(true) - apply := func(_ context.Context, raw, _ []byte) Report { - applied.Add(1) - var d struct{ Order } - _ = json.Unmarshal(raw, &d) - return Report{Declared: digest(raw), Order: d.Order, Applied: []string{"a.x", "b.x"}, + facts := func() Report { + return Report{Applied: []string{"a.x", "b.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: failB.Load(), Detail: "b.x failed: no space"}}, Health: healthOf(map[string]string{"a": StateHealthy, "b": StateHealthy}), Windows: []Window{{Step: "backup", Holds: []string{"db"}}}} } - q := aQueue(m, apply, &keptInMemory{}) + q := &Session{Membership: m, Timeout: 2 * time.Second} q.attach(context.Background(), OverNATS{Conn: conn, JS: js}) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - defer func() { stop(); <-worker }() + ctx := context.Background() revisions := func() map[string]uint64 { out := map[string]uint64{} @@ -62,14 +55,14 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { } return out } - arrives := func(what string, id string, seq int64) { - want := applied.Load() + 1 - q.Deliver(ordered(t, key, id, 7, seq)) - eventually(t, what, func() bool { return applied.Load() >= want }) + arrives := func(what string) { + if !q.SayFacts(ctx, facts()) { + t.Fatal(what + ": the facts were not taken") + } time.Sleep(300 * time.Millisecond) } - arrives("the first apply", "one", 1) - eventually(t, "the three keys were never written", func() bool { return len(revisions()) == 3 }) + arrives("the first facts") + eventuallyHeld(t, "the three keys were never written", func() bool { return len(revisions()) == 3 }) first := revisions() a, _ := kv.Get(node + ".a") @@ -87,7 +80,7 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { t.Fatalf("the node key holds %+v (%v)", hn, err) } - arrives("the second apply", "two", 2) + arrives("the second facts") time.Sleep(200 * time.Millisecond) for name, rev := range revisions() { if rev != first[name] { @@ -96,8 +89,8 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { } failB.Store(false) - arrives("the third apply", "three", 3) - eventually(t, "b was never updated", func() bool { return revisions()["b"] > first["b"] }) + arrives("the third facts") + eventuallyHeld(t, "b was never updated", func() bool { return revisions()["b"] > first["b"] }) if now := revisions(); now["a"] != first["a"] || now["node"] != first["node"] { t.Errorf("a key that did not change moved: %v from %v", now, first) } diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 152e8aa..2e6942d 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -126,15 +126,15 @@ func healthOf(states map[string]string) *Health { } func appliedModules() Report { - return Report{Declared: "d1", Applied: []string{"a.x", "b.x"}, + return Report{Applied: []string{"a.x", "b.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}}, Health: healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})} } -func aHaveQueue(l *haveLink) (*Queue, *[]string) { +func aHaveQueue(l *haveLink) (*Session, *[]string) { var said []string var mu sync.Mutex - q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) { + q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) { mu.Lock() said = append(said, s) mu.Unlock() @@ -148,7 +148,7 @@ func aHaveQueue(l *haveLink) (*Queue, *[]string) { func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) a, b := l.have(t, "n1.a"), l.have(t, "n1.b") if a.Phase != delivery.PhaseRunning || a.Build != "b-1" || a.Health != delivery.HealthOK { t.Errorf("a: %+v", a) @@ -168,7 +168,7 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-0", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.kv["n1.a"].rev != 1 { t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev) } @@ -176,7 +176,7 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev) } before := l.setCount() - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.setCount() != before { t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before) } @@ -186,9 +186,9 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { func TestAModulesHealthFollowsTheLook(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - rep := Report{Declared: "d1", Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"}, + rep := Report{Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"}, {Module: "c", Build: "b"}, {Module: "d", Build: "b"}}} - q.tell(context.Background(), rep, applied) + q.SayFacts(context.Background(), rep) for _, m := range []string{"a", "b", "c", "d"} { if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown { t.Errorf("%s with no look is %s, want unknown", m, h) @@ -230,7 +230,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { rep.Strays = []Stray{{Kind: "container", Name: "old"}} rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}} rep.Profile = map[string]any{"cores": 8} - q.tell(context.Background(), rep, applied) + q.SayFacts(context.Background(), rep) if len(l.events) < 2 || l.events[0] != "report" { t.Fatalf("the report was not first: %v", l.events) } @@ -255,7 +255,7 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) - if !q.tell(context.Background(), appliedModules(), applied) { + if !q.SayFacts(context.Background(), appliedModules()) { t.Fatal("the report did not go out") } if len(l.said()) != 1 { @@ -267,7 +267,7 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { l.hmu.Lock() l.failing = false l.hmu.Unlock() - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.have(t, "n1.a").Phase != delivery.PhaseRunning { t.Error("the next apply did not write again") } @@ -276,39 +276,19 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { // A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong. func TestNoBucketAndNoFactsWriteNothing(t *testing.T) { plain := newQuietLink() - q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second} + q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second} q.attach(context.Background(), plain) - if !q.tell(context.Background(), appliedModules(), applied) { + if !q.SayFacts(context.Background(), appliedModules()) { t.Fatal("the report did not go out") } l := newHaveLink() q2, _ := aHaveQueue(l) - q2.tell(context.Background(), Report{Refused: "no"}, applied) + q2.SayFacts(context.Background(), Report{Refused: "no"}) if l.setCount() != 0 { t.Errorf("a refusal wrote %d keys", l.setCount()) } } -// 1132/AC5: a reconcile whose report is not news still writes the have keys that differ, so a node-engine that -// restarted writes them again; and writes nothing when they are the same. -func TestASilentReconcileStillWritesTheHaves(t *testing.T) { - l := newHaveLink() - q, _ := aHaveQueue(l) - q.Reconcile = func(context.Context) (Report, bool) { return appliedModules(), true } - q.reconcile(context.Background(), false) - if len(l.said()) != 0 { - t.Fatalf("a reconcile that is not news said %d reports", len(l.said())) - } - if l.have(t, "n1.a").Phase != delivery.PhaseRunning { - t.Error("the reconcile wrote no have") - } - before := l.setCount() - q.reconcile(context.Background(), false) - if l.setCount() != before { - t.Errorf("an unchanged reconcile wrote %d keys", l.setCount()-before) - } -} - // 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that // fails ends the pass, and is said once. func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { @@ -321,7 +301,7 @@ func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { q, said := aHaveQueue(l) q.Timeout = 100 * time.Millisecond start := time.Now() - if !q.tell(context.Background(), appliedModules(), applied) { + if !q.SayFacts(context.Background(), appliedModules()) { t.Fatalf("%s: the report did not go out", name) } if took := time.Since(start); took > time.Second { @@ -334,14 +314,14 @@ func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.binds != 1 || l.setTries != 1 || len(*said) != 1 { t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said) } } func onlyA() Report { - return Report{Declared: "d2", Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}}, + return Report{Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}}, Health: healthOf(map[string]string{"a": StateHealthy})} } @@ -350,9 +330,9 @@ func onlyA() Report { func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) b := l.have(t, "n1.b") if b.Phase != delivery.PhaseRemoved || b.Build != "b-2" { t.Errorf("b: %+v", b) @@ -364,7 +344,7 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev) } before := l.setCount() - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) if l.setCount() != before { t.Errorf("a second apply without b wrote %d keys, removed is written once", l.setCount()-before) } @@ -374,13 +354,13 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { func TestARemovedModuleAddedBackIsWrittenWithItsNewPhase(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) - q.tell(context.Background(), onlyA(), applied) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) + q.SayFacts(context.Background(), onlyA()) + q.SayFacts(context.Background(), appliedModules()) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseFailed || b.Build != "b-2" { t.Errorf("b after it came back: %+v", b) } - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved { t.Errorf("b removed again: %+v", b) } @@ -392,19 +372,19 @@ func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) { l := newHaveLink() l.put(t, "n1.c", delivery.Have{V: delivery.Version, Build: "b-9", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), appliedModules()) + q.SayFacts(context.Background(), onlyA()) if c := l.have(t, "n1.c"); c.Phase != delivery.PhaseRunning || l.kv["n1.c"].rev != 1 { t.Errorf("c, which this node-engine never wrote, changed: %+v at %d", c, l.kv["n1.c"].rev) } // a key that vanished from the bucket is not created by the removal l2 := newHaveLink() q2, _ := aHaveQueue(l2) - q2.tell(context.Background(), appliedModules(), applied) + q2.SayFacts(context.Background(), appliedModules()) l2.hmu.Lock() delete(l2.kv, "n1.b") l2.hmu.Unlock() - q2.tell(context.Background(), onlyA(), applied) + q2.SayFacts(context.Background(), onlyA()) if _, there := l2.kv["n1.b"]; there { t.Error("the removal created a key that was not there") } @@ -414,15 +394,15 @@ func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) { func TestARefusedRemovalIsWrittenByTheNextPass(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) l.hmu.Lock() l.failing = true l.hmu.Unlock() - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) l.hmu.Lock() l.failing = false l.hmu.Unlock() - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved { t.Errorf("b: %+v", b) } @@ -437,7 +417,7 @@ func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T) q, _ := aHaveQueue(l) // fresh: nothing written by this process r := onlyA() r.Before = []string{"a", "b", "c"} - q.tell(context.Background(), r, applied) + q.SayFacts(context.Background(), r) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved { t.Errorf("b: %+v", b) } @@ -448,3 +428,36 @@ func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T) t.Errorf("a was written, revision %d", l.kv["n1.a"].rev) } } + +// R6-A AC7: the first have this node-engine writes — or finds already saying what it would write — is told to the +// proof, which proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9); a write the bucket refuses is not. +func TestTheFirstHaveWrittenIsToldToTheProof(t *testing.T) { + l := newHaveLink() + l.failing = true + q, _ := aHaveQueue(l) + var told int + q.HaveWritten = func() { told++ } + q.SayFacts(context.Background(), appliedModules()) + if told != 0 { + t.Fatalf("a refused write proved the build %d time(s)", told) + } + l.hmu.Lock() + l.failing = false + l.hmu.Unlock() + q.SayFacts(context.Background(), appliedModules()) + if told == 0 { + t.Fatal("a have written under this build was not told") + } + + // The bucket already holds what this node-engine would write: a restart with nothing changed still proves it. + same := newHaveLink() + q2, _ := aHaveQueue(same) + q2.SayFacts(context.Background(), appliedModules()) + again, _ := aHaveQueue(same) + told = 0 + again.HaveWritten = func() { told++ } + again.SayFacts(context.Background(), appliedModules()) + if told == 0 { + t.Fatal("a have already saying the same was not told") + } +} diff --git a/internal/link/health_test.go b/internal/link/health_test.go index 3132c4b..5ef80ef 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -23,7 +23,7 @@ func (l *sayingLink) Health(_ context.Context, node string, body []byte) error { } func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { - q := &Queue{Membership: Membership{Node: "anchor"}} + q := &Session{Membership: Membership{Node: "anchor"}} h := Health{Contract: LivenessContract, At: time.Now().UTC(), Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy, Reason: ReasonRestarting}}} if q.SayHealth(t.Context(), h) { @@ -77,7 +77,7 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) { // the link was up, because the NATS link could not Ask). With no link open, and with a link that cannot // ask, the check fails and says which. func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) { - q := &Queue{Membership: Membership{Node: "anchor"}} + q := &Session{Membership: Membership{Node: "anchor"}} if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" { t.Fatalf("with no link open: %v", err) } diff --git a/internal/link/hearing.go b/internal/link/hearing.go index e1f4503..0301409 100644 --- a/internal/link/hearing.go +++ b/internal/link/hearing.go @@ -8,9 +8,8 @@ import ( // What a host hears, as the host's own words for it. // -// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and the -// declarations that arrive. The run loop reads its own words for a declaration rather than the -// client library's delivery type, so nothing past this file knows what carried it. +// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and what the controller asks of this +// node. A node hears no declaration: it follows its wants (novox/hq ADR 0539). // // **The host still imports nothing of the mesh's own** (novox/hq ADR 0005). This is its own // interface over its own libraries, and it agrees with the controller only because a conformance @@ -21,12 +20,9 @@ import ( // One interface rather than two, because dialling once is what keeps both halves of a node on the // same connection. type Link interface { - // Bus is what this node says: what it applied, and that it is here. + // Bus is what this node says: its facts, and that it is here. Bus - // Declarations is what the mesh tells this node to be. - Declarations() <-chan Declaration - // Lost says the link ended, and why. // // **Read rather than discovered.** A node that finds out by noticing silence is a node that @@ -38,31 +34,12 @@ type Link interface { Close() } -// Asked is a link that hears the mesh asking this node to say again what it last applied (novox/hq -// to-be 45 §6, the `report` verb). Optional: a link without it is never asked, and the mesh then sends -// the current declaration again instead. +// Asked is a link that hears the mesh asking this node to say its facts again. Optional: a link without it is never +// asked. type Asked interface { AskedToReport() <-chan struct{} } -// Declaration is one thing the mesh told this node to be. -// -// **Handled, once — after the report is published.** A node that dies between applying and -// reporting leaves the declaration with the mesh and applies it again on return, which is safe -// because applying is reconciliation: it converges rather than repeating. -// -// There is one way of being done rather than two. A declaration set aside because a newer arrived -// with it is settled exactly as an applied one is, and the difference between them is a fact the -// *report* carries — a second method here would be a distinction the bus does not make. -type Declaration interface { - // Body is the signed declaration as it arrived, bytes unchanged: a node verifies what it - // received rather than what it re-encoded. - Body() []byte - - // Handled settles it. Called after the report for it has been published, either way. - Handled() error -} - // Open opens this node's link to its mesh. // // Named Open rather than Dial because Dial is this package's raw TLS dial, which the enrolment path diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index ba8d84b..da68fa2 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -11,37 +11,16 @@ import ( "github.com/nats-io/nats.go" ) -// The host's link on the bus being built. -// -// Two things a host does here that it cannot do on the other bus, and one it must not try. -// -// **It declares nothing.** On the bus the mesh has, a host declares its own queue on connecting, -// because a queue that is not there means a node that hears nothing. Here the object it reads -// through is a durable consumer, and a host's account reaches no part of the JetStream API — by -// design, because the controller is the only writer of consumer definitions (design 25 §3). So the -// host **binds** to a consumer the controller made when this node enrolled, and a missing one is -// said as what it is rather than quietly created with whatever configuration this client happens to -// default to. -// -// **It gets order for free, and keeps the drain anyway.** The declaration subject is last-per-subject -// (design 29 §4), so a node that was away receives exactly the current declaration rather than a -// queue of superseded ones, and the stream's sequence orders them definitively — the wire-level -// answer to novox/hq issue 107. What the drain in run.go still answers is the live case: three -// pushes to a *connected* node are three deliveries whatever the stream later retains. +// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration +// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts. // EnrolSubject is where a joining machine asks. One subject for every node, because a machine // enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is // the whole of what its enrolment user may do. const EnrolSubject = "mesh.control.enrol" -// DeclareSubject is where this node's declaration lands. Its own, and no other node's: a host's -// account subscribes exactly this and the subject is the authority on which node a declaration is -// for. -func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" } - -// AskReportSubject is where the mesh asks this node to say again what it last applied (novox/hq to-be 45 -// §6): its own, on core NATS and off any stream. The answer is an ordinary report on its report subject, -// the one thing it already says — nobody's inbox is answered. +// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream. +// The answer is the facts event on its report subject — nobody's inbox is answered. func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" } // natsURL is a bus address as the client wants it. A membership records host and port, because that @@ -58,7 +37,6 @@ func natsURL(address string) string { type natsLink struct { conn *nats.Conn js nats.JetStreamContext - sub *nats.Subscription asking *nats.Subscription asked chan struct{} handing *nats.Subscription @@ -72,9 +50,8 @@ type natsLink struct { moving *nats.Subscription node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). - signer ed25519.PublicKey - arrived chan Declaration - lost chan error + signer ed25519.PublicKey + lost chan error } func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, error) { @@ -123,27 +100,10 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e l := &natsLink{ conn: conn, js: js, node: m.Node, signer: m.Signer, - arrived: make(chan Declaration, drainDepth), - lost: make(chan error, 1), - asked: make(chan struct{}, 1), + lost: make(chan error, 1), + asked: make(chan struct{}, 1), } - // Bound to the consumer the controller made for this node, named after the node because that is - // what the node's own ack grant allows (`$JS.ACK.NODES..>`). - feed := make(chan *nats.Msg, drainDepth) - // The subject as well as the binding: the client checks what is asked for against the - // consumer's own filter, and an empty subject is refused rather than taken to mean "whatever - // that consumer delivers". - sub, err := js.ChanSubscribe(DeclareSubject(m.Node), feed, nats.Bind("NODES", m.Node)) - if err != nil { - conn.Close() - return nil, fmt.Errorf( - "this node cannot read its declarations: %w. The mesh creates that when a node enrols, "+ - "and a host may not create one itself — so this is the mesh's to answer, not this "+ - "machine's", err) - } - l.sub = sub - l.hearAsks() conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { @@ -159,29 +119,6 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e } }) - go func() { - defer close(l.arrived) - for { - select { - case <-ctx.Done(): - return - case msg, ok := <-feed: - if !ok { - select { - case l.lost <- errors.New("the bus stopped delivering"): - default: - } - return - } - select { - case l.arrived <- natsDeclaration{msg}: - case <-ctx.Done(): - return - } - } - } - }() - return l, nil } @@ -299,14 +236,10 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) { } } -func (l *natsLink) Declarations() <-chan Declaration { return l.arrived } -func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } -func (l *natsLink) Lost() <-chan error { return l.lost } +func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } +func (l *natsLink) Lost() <-chan error { return l.lost } func (l *natsLink) Close() { - if l.sub != nil { - _ = l.sub.Unsubscribe() - } if l.asking != nil { _ = l.asking.Unsubscribe() } @@ -375,12 +308,3 @@ var ( _ Asker = (*natsLink)(nil) _ Asked = (*natsLink)(nil) ) - -// natsDeclaration is one declaration off the NODES stream. -type natsDeclaration struct{ msg *nats.Msg } - -func (d natsDeclaration) Body() []byte { return d.msg.Data } - -// Handled acknowledges it. The ack goes to this node's own ack subject, which is the one thing -// besides its reports a node's account may publish. -func (d natsDeclaration) Handled() error { return d.msg.Ack() } diff --git a/internal/link/hearing_nats_test.go b/internal/link/hearing_nats_test.go deleted file mode 100644 index f32bc88..0000000 --- a/internal/link/hearing_nats_test.go +++ /dev/null @@ -1,337 +0,0 @@ -package link - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "fmt" - "os" - "reflect" - "testing" - "time" - - "github.com/nats-io/nats.go" -) - -// The host's link against a real server, because every claim here is about one. -// -// Whether binding to a consumer the host did not create works, whether a declaration on the node's -// own subject arrives, whether acknowledging it removes it from the consumer's pending — none of -// that can be reasoned out, and the first two are the ones that would leave a node silently hearing -// nothing: -// -// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js -// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNats - -func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) { - t.Helper() - url := os.Getenv("MESH_TEST_NATS") - if url == "" { - t.Skip("MESH_TEST_NATS unset") - } - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - // **Ensured and purged, not deleted and recreated.** Delete-then-add looked like a reset and is - // not one: a test that did that inherited the previous test's messages, and the symptom was a - // declaration counted as delivered twice — which reads as a redelivery bug in the code under - // test rather than as a dirty stream. Purge is defined to empty a stream; recreating one is a - // race with the server's own teardown. - for _, want := range []*nats.StreamConfig{ - {Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, MaxMsgsPerSubject: 1}, - {Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, - Retention: nats.WorkQueuePolicy}, - } { - if _, err := js.StreamInfo(want.Name); err != nil { - if _, err := js.AddStream(want); err != nil { - t.Fatal(err) - } - } - if err := js.PurgeStream(want.Name); err != nil { - t.Fatal(err) - } - } - return conn, js -} - -// theMeshMakes is the consumer the controller creates when a node enrols. Made here by the test -// because the host may not: its account reaches no part of the JetStream API, which is the whole -// reason this binds rather than subscribes. -// -// Removed afterwards, and each test names its own node: two tests sharing a consumer name share its -// delivery count and its pending list, and the first thing that goes wrong reads as a fault in the -// host rather than in the test beside it. -func theMeshMakes(t *testing.T, js nats.JetStreamContext, node string) { - t.Helper() - t.Cleanup(func() { _ = js.DeleteConsumer("NODES", node) }) - if _, err := js.AddConsumer("NODES", &nats.ConsumerConfig{ - Durable: node, - FilterSubject: DeclareSubject(node), - AckPolicy: nats.AckExplicitPolicy, - AckWait: 300 * time.Second, - DeliverSubject: "_DELIVER." + node, - }); err != nil { - t.Fatal(err) - } -} - -func signedBy(t *testing.T, key ed25519.PrivateKey, declaration []byte) []byte { - t.Helper() - body, err := json.Marshal(Signed{ - Declaration: declaration, Signature: ed25519.Sign(key, declaration), - }) - if err != nil { - t.Fatal(err) - } - return body -} - -// A declaration on this node's own subject reaches the host, is applied, and acknowledging it -// empties the consumer — which is what tells the mesh the node has it. -func TestNatsADeclarationReachesTheHostAndIsSettled(t *testing.T) { - conn, js := aBus(t) - const node = "settling" - theMeshMakes(t, js, node) - - public, private, _ := ed25519.GenerateKey(nil) - m := Membership{Node: node, Signer: public} - - // Dialled directly rather than through Open: the test server has no TLS, and what is being - // checked is the subscription and the settling, not the pin — which PinnedConfig owns and its - // own tests cover. - l := &natsLink{conn: conn, js: js, node: node, - arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)} - feed := make(chan *nats.Msg, drainDepth) - sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node)) - if err != nil { - t.Fatalf("the host could not bind to the consumer the mesh made for it: %v", err) - } - defer func() { _ = sub.Unsubscribe() }() - go func() { - for msg := range feed { - l.arrived <- natsDeclaration{msg} - } - }() - - if _, err := js.Publish(DeclareSubject(node), - signedBy(t, private, []byte(`{"declared":"d1"}`))); err != nil { - t.Fatal(err) - } - - select { - case d := <-l.Declarations(): - report := handleBody(context.Background(), m, d.Body(), - func(context.Context, []byte, []byte) Report { - return Report{Applied: []string{"store"}} - }) - if report.Refused != "" { - t.Fatalf("a declaration the mesh signed was refused: %s", report.Refused) - } - if err := d.Handled(); err != nil { - t.Fatalf("the node could not acknowledge its own declaration: %v", err) - } - case <-time.After(8 * time.Second): - t.Fatal("no declaration reached the host") - } - - // **Nothing pending is the property**; a delivery count is not. Delivery is at-least-once by - // design, so pinning "delivered exactly once" would be asserting something the mesh does not - // rely on. What matters is that the acknowledgement landed, so the mesh can tell the node has - // it — and that no redelivery was needed to get there, which is what would say the node was - // too slow to answer for its own ack wait. - deadline := time.Now().Add(5 * time.Second) - var last string - for time.Now().Before(deadline) { - info, err := js.ConsumerInfo("NODES", node) - switch { - case err != nil: - last = err.Error() - case info.NumAckPending == 0 && info.NumRedelivered == 0: - return - default: - last = fmt.Sprintf("pending %d, redelivered %d", info.NumAckPending, info.NumRedelivered) - } - time.Sleep(20 * time.Millisecond) - } - t.Fatalf("the declaration was not settled, so the mesh cannot tell the node has it: %s", last) -} - -// **A node that was away gets exactly the current declaration and nothing older.** Three pushed -// while nothing is listening leave one on the stream, and it is the newest — the wire-level answer -// to novox/hq issue 107, and the half of the drain that stops being the host's problem. -func TestNatsANodeThatWasAwayGetsOnlyTheNewest(t *testing.T) { - _, js := aBus(t) - const node = "returning" - _, private, _ := ed25519.GenerateKey(nil) - - for _, id := range []string{"d1", "d2", "d3"} { - if _, err := js.Publish(DeclareSubject(node), - signedBy(t, private, []byte(`{"declared":"`+id+`"}`))); err != nil { - t.Fatal(err) - } - } - info, err := js.StreamInfo("NODES") - if err != nil { - t.Fatal(err) - } - if info.State.Msgs != 1 { - t.Fatalf("%d declarations survived for one node; a node that was away would apply a backlog "+ - "of things nobody wants any more", info.State.Msgs) - } - - theMeshMakes(t, js, node) - feed := make(chan *nats.Msg, drainDepth) - sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node)) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - - select { - case msg := <-feed: - if want := declaredIn(signedBy(t, private, []byte(`{"declared":"d3"}`))); declaredIn(msg.Data) != want { - t.Fatalf("the node was given %q rather than the newest", msg.Data) - } - case <-time.After(8 * time.Second): - t.Fatal("the node that was away was given nothing") - } -} - -// A report goes through the stream: it must survive the controller's store restarting. -func TestNatsAReportIsKept(t *testing.T) { - conn, js := aBus(t) - bus := OverNATS{Conn: conn, JS: js} - ctx := context.Background() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1"}) - if err := bus.Report(ctx, "anchor", body); err != nil { - t.Fatal(err) - } - - info, err := js.StreamInfo("CONTROL") - if err != nil { - t.Fatal(err) - } - if info.State.Msgs != 1 { - t.Fatalf("%d messages were kept; a report must be, once", info.State.Msgs) - } -} - -// **One apply queue, against a real bus** (novox/hq to-be 45 §6, R2). A declaration is being applied -// when two more are pushed and the five-minute reconcile comes due: the worker applies the one in -// hand, then only the newest — the reconcile is that apply — and the mesh's stream holds one account -// per apply, each naming the order it applied, numbered in the order they were made. Every -// declaration delivered is settled. -func TestNatsTheQueueAppliesTheNewestOnceAndReportsInOrder(t *testing.T) { - conn, js := aBus(t) - const node = "queueing" - theMeshMakes(t, js, node) - public, private, _ := ed25519.GenerateKey(nil) - m := Membership{Node: node, Signer: public} - - ctx, stop := context.WithCancel(context.Background()) - defer stop() - l := &natsLink{conn: conn, js: js, node: node, - arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)} - feed := make(chan *nats.Msg, drainDepth) - sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node)) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - go func() { - for msg := range feed { - l.arrived <- natsDeclaration{msg} - } - }() - reports, err := js.SubscribeSync(ReportSubject(node), nats.BindStream("CONTROL")) - if err != nil { - t.Fatal(err) - } - defer func() { _ = reports.Unsubscribe() }() - - a := &applies{} - inFirst, release := make(chan struct{}, 1), make(chan struct{}) - reconciled := 0 - q := &Queue{Membership: m, Timeout: 5 * time.Second, Unsaid: &keptInMemory{}, - Apply: func(ctx context.Context, raw, sig []byte) Report { - r := a.apply(ctx, raw, sig) - if r.Sequence == 1 { - inFirst <- struct{}{} - <-release - } - return r - }, - Reconcile: func(context.Context) (Report, bool) { reconciled++; return Report{}, false }, - } - worker := runWorker(ctx, q) - go func() { _ = serve(ctx, l, m, q, nil, 5*time.Second) }() - - push := func(id string, sequence int64) { - inner, _ := json.Marshal(map[string]any{"declaration": 1, "epoch": 7, "sequence": sequence, - "resources": []any{map[string]any{"id": id}}}) - if _, err := js.Publish(DeclareSubject(node), signedBy(t, private, inner)); err != nil { - t.Fatal(err) - } - } - push("one", 1) - select { - case <-inFirst: - case <-time.After(8 * time.Second): - t.Fatal("the first declaration never reached the worker") - } - push("two", 2) - push("three", 3) - q.ReconcileDue() - time.Sleep(2 * drainWindow) // the link gathers both and hands them to the queue - close(release) - - var heard []Report - for len(accounts(heard)) < 2 { - msg, err := reports.NextMsg(8 * time.Second) - if err != nil { - t.Fatalf("the mesh heard %+v and then nothing: %v", heard, err) - } - var r Report - if err := json.Unmarshal(msg.Data, &r); err != nil { - t.Fatal(err) - } - heard = append(heard, r) - _ = msg.Ack() - } - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"one", "three"}) { - t.Fatalf("applied %v; the one in hand and then only the newest were wanted", got) - } - if reconciled != 0 { - t.Fatal("the reconcile applied beside the delivery it was due with") - } - accounted := accounts(heard) - if accounted[0].Sequence != 1 || accounted[1].Sequence != 3 || accounted[1].Epoch != 7 { - t.Fatalf("the accounts do not name what was applied: %+v", accounted) - } - for i := 1; i < len(heard); i++ { - if heard[i].ReportSequence <= heard[i-1].ReportSequence { - t.Fatalf("the reports left out of the order they were made: %+v", heard) - } - } - deadline := time.Now().Add(5 * time.Second) - for { - info, err := js.ConsumerInfo("NODES", node) - if err == nil && info.NumAckPending == 0 { - break - } - if time.Now().After(deadline) { - t.Fatalf("a delivered declaration was left unsettled: %+v %v", info, err) - } - time.Sleep(20 * time.Millisecond) - } -} diff --git a/internal/link/held_note_test.go b/internal/link/held_note_test.go deleted file mode 100644 index 43b9953..0000000 --- a/internal/link/held_note_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package link - -import ( - "strings" - "testing" -) - -// The count that did not add up was the only symptom sixteen held resources had, and reading it meant -// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did -// says what it did not, too. - -func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) { - got := heldNote([]Held{ - {ID: "ca", Module: "route-proxy", Kind: "directory"}, - {ID: "certs", Module: "route-proxy", Kind: "directory"}, - {ID: "server", Module: "route-proxy", Kind: "container"}, - {ID: "mail", Module: "mailu", Kind: "container"}, - }) - if !strings.Contains(got, "4 held") { - t.Fatalf("the count of what was held is not in the line: %q", got) - } - // The module is the thing an operator can act on: `take` takes a module. - if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") { - t.Fatalf("the line does not break the holds down by module: %q", got) - } - // Ordered, so two machines holding the same things read the same and a diff of two reports is - // about what changed. - if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") { - t.Fatalf("modules are not in a stated order: %q", got) - } - // It says why, because "held" alone reads as a failure and this is correct behaviour. - if !strings.Contains(got, "taken") { - t.Fatalf("the line does not say a hold ends when the module is taken: %q", got) - } -} - -func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) { - // A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of - // them is how a line stops being read. - if got := heldNote(nil); got != "" { - t.Fatalf("an apply with no holds added %q to its line", got) - } - if got := heldNote([]Held{}); got != "" { - t.Fatalf("an apply with no holds added %q to its line", got) - } -} diff --git a/internal/link/messages.go b/internal/link/messages.go index 406c75d..548dafd 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -18,22 +18,6 @@ const ( KeyReport = "report" ) -// Signed is a declaration and the signature over it. -// -// novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is -// verified by its signature, every time**. The two are different questions — a node connects to -// the broker and takes instruction from the control plane behind it, and pinning only the first -// would make the second transitive. -// -// The signature is over Declaration exactly as it arrived, bytes unchanged. Re-encoding before -// verifying would mean checking a signature over something other than what was sent, and any -// difference in key order or spacing would break it — so the raw message is what is signed and -// what is checked. -type Signed struct { - Declaration []byte `json:"declaration"` - Signature []byte `json:"signature"` -} - // Report is what a node says after applying, and it is a statement rather than a write. // // A node states; the context that owns the data writes (novox/hq ADR 0006). The difference is the @@ -51,14 +35,6 @@ type Report struct { // Refused is set when the declaration was rejected whole rather than applied in part. Refused string `json:"refused,omitempty"` - // Superseded names the newer declaration this one was set aside for, unapplied. - // - // A machine asked to be five successive things becomes the last one (novox/hq issue 031): - // when several declarations are waiting, the host applies the newest and acknowledges the - // rest without applying them. Each of those is still reported, because silence reads as a - // machine that ignored an instruction and "applied" would be a lie — this is the third word. - Superseded string `json:"superseded,omitempty"` - // Carried are the machine's ports held by what this host raised from its own bundle. // // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node @@ -70,62 +46,13 @@ type Report struct { // machine saying what is true of it, not asking for anything. Carried []int `json:"carried,omitempty"` - // Declared is the digest of the declaration this report is about — sha256 of the exact bytes - // the mesh sent, which the mesh recorded when it sent them. - // - // **Which declaration, not when.** The mesh compared its send time to this report's arrival - // to decide whether a machine had caught up, and lost the race it invited: an apply started - // under the previous declaration finishes after the next one is sent, its report lands newer - // than the send, and the machine reads as caught up with words it has not read yet. Clocks - // cannot answer "which"; the digest is the answer itself. - Declared string `json:"declared,omitempty"` - - // Order is where the declaration this report is about stands — its `epoch` and `sequence`, as - // the declaration carried them — so the mesh orders reports by what they are about rather than - // by when they arrived (novox/hq to-be 45 §6). On the wire as two top-level keys beside - // `declared`; absent for a declaration that claimed no order, and for a report about none. + // Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand + // among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine, + // the sequence one higher for every facts event, so the controller refuses older facts by this order. On the + // wire as three top-level keys. + Writer string `json:"writer,omitempty"` Order - // ReportSequence is this node-engine's own order among everything it reports: one higher for - // every report it makes, kept on disk so it goes on increasing across restarts and self-updates - // (to-be 45 §6). With the declaration's order it lets the mesh refuse an older report about the - // same declaration — a reconcile's account overtaking the apply that followed it (novox/hq issue - // 267) is then refused by number rather than set aside by digest. A report said again because - // it never reached the mesh (issue 264) carries the number it was made with. - // - // Zero claims no order: every report an older host makes, and a one-shot report a command makes - // (a rekey). **Its presence also says this host reads a declaration's `epoch`**, which is how the - // mesh knows it may send one: an older host decodes a declaration strictly and refuses a key it - // does not know. - ReportSequence int64 `json:"report_sequence,omitempty"` - - // OlderThan is set on a report refusing a declaration older than one this node has applied: the - // order of the newer declaration it holds (to-be 45 §6, rule 2). The refused declaration is the - // report's own Declared and Order, and Refused says it in words. Applying it would make the - // machine into something a controller that lost its lease said, after one that holds it. - OlderThan *Order `json:"older_than,omitempty"` - - // RefusedOlder is how many declarations this node-engine has refused as older, ever — kept on - // disk with the report sequence. On every report, so a count missed with a lost refusal is still - // read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts. - RefusedOlder int64 `json:"refused_older,omitempty"` - - // ReadsGeneration says this node-engine reads a declaration's assignment generation and its - // put-back mark (novox/hq issue 234), on every report it makes: the controller sends them only to a - // machine that said so, because an older node-engine decodes strictly and refuses a key it does not - // know, whole. A node-engine rolled back says so no longer. - ReadsGeneration bool `json:"reads_generation,omitempty"` - - // ReadsModules says this node-engine reads a resource's `module` and a declaration's `builds` (novox/hq ADR 0429 - // §4, issue 1132), on every report it makes, beside ReadsGeneration: the controller sends them only to a machine - // that said so, for the same reason. - ReadsModules bool `json:"reads_modules,omitempty"` - - // OlderGeneration is set on a report refusing a declaration composed from an older assignment - // generation than this machine applied (novox/hq issue 234). The refused declaration is the report's - // own Declared and Order, so the controller can name who sent it from its record of sends. - OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"` - // Held is what this adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` @@ -647,52 +574,14 @@ type Rollback struct { At time.Time `json:"at"` } -// Order is where a declaration stands among everything the mesh has sent this node (novox/hq to-be -// 45 §6): the controller's lease **epoch** it was sent under, and its **sequence** — one higher for -// every send to this node (04-ISSUES/107). The declaration carries both inside what is signed, as -// top-level `epoch` and `sequence` beside `declaration`; a report carries back the pair of the -// declaration it is about. -// -// Zero in either is "no order claimed", never "first": every declaration an older controller sent, -// and the bundle genesis applies. +// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its +// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one +// with nothing kept on disk. type Order struct { Epoch int64 `json:"epoch,omitempty"` Sequence int64 `json:"sequence,omitempty"` } -// Older says whether a declaration of this order is older than one of order `than`, which this node -// has applied — the one rule by which a node-engine refuses a declaration (to-be 45 §6: "older epoch; -// same epoch, lower sequence"). -// -// **Only when both claim an epoch.** A declaration with none is an older controller's — or a -// controller rolled back to a build from before the lease — and one kept with none is what this host -// held before any controller had a lease. Neither has an order to compare, and refusing on a guess -// would strand the machine the moment the controller is older than the host: without an epoch, today's -// behaviour stands. A higher epoch is a new lease holder and is never older, whatever its sequence. -func (o Order) Older(than Order) bool { - if o.Epoch <= 0 || than.Epoch <= 0 { - return false - } - if o.Epoch != than.Epoch { - return o.Epoch < than.Epoch - } - return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence -} - -// Supersedes says whether a declaration of order `o`, arriving after one of order `before`, takes its -// place among what is waiting to be applied. By epoch when both claim one and they differ, then by -// sequence when both claim one, and by arrival when they do not — what the drain had to go on before -// declarations said where they stand (04-ISSUES/107). -func (o Order) Supersedes(before Order) bool { - if o.Epoch > 0 && before.Epoch > 0 && o.Epoch != before.Epoch { - return o.Epoch > before.Epoch - } - if o.Sequence > 0 && before.Sequence > 0 { - return o.Sequence >= before.Sequence - } - return true -} - // Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a // Stringer promoted onto every report would print each one as its order alone. func (o Order) Words() string { @@ -842,13 +731,6 @@ type FoundFirewall struct { RetiredBy string `json:"retired_by,omitempty"` } -// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that -// generation, and the highest this machine applied (novox/hq issue 234). -type GenerationRefusal struct { - Generation int64 `json:"generation"` - Applied int64 `json:"applied"` -} - // ModuleFact is how one module's part of an apply came out: the build the node was sent for it, and whether every // resource of it applied. Detail names the failing resource and its error (issue 1132). type ModuleFact struct { diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index d2cb235..b27f39b 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -1,115 +1,10 @@ package link import ( - "context" - "crypto/ed25519" "encoding/json" "testing" ) -// verified runs what Run does to a delivery body, without a broker: unmarshal, check the -// signature, and only then apply. Isolating it keeps this test about the check rather than about -// the bus, which is tested against a real one in the lab. -func verified(t *testing.T, signer ed25519.PublicKey, body []byte) (Report, bool) { - t.Helper() - applied := false - report := handleBody(context.Background(), Membership{Node: "anchor", Signer: signer}, body, - func(context.Context, []byte, []byte) Report { - applied = true - return Report{Applied: []string{"something"}} - }) - return report, applied -} - -func signedBody(t *testing.T, private ed25519.PrivateKey, declaration string) []byte { - t.Helper() - raw, err := json.Marshal(Signed{ - Declaration: []byte(declaration), - Signature: ed25519.Sign(private, []byte(declaration)), - }) - if err != nil { - t.Fatal(err) - } - return raw -} - -func TestTheMeshsOwnDeclarationIsApplied(t *testing.T) { - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - report, applied := verified(t, public, signedBody(t, private, `{"declaration":1}`)) - if !applied { - t.Fatalf("a declaration the mesh signed was not applied: %s", report.Refused) - } -} - -// Defends novox/hq ADR 0002: everything reaching a node arrives over the broker — and therefore -// ADR 0004's consequence, that the broker is not trusted to say who is speaking. -// -// A transport nobody authenticates per-message would let whatever holds the connection attribute -// a declaration to any node it liked. -func TestAForgedDeclarationIsNeverApplied(t *testing.T) { - // The check that stands between "the mesh changes this machine" and "anybody does". The host - // applies whatever the link delivers, so a forged declaration is the whole machine. - public, _, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - _, other, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - - report, applied := verified(t, public, signedBody(t, other, `{"declaration":1}`)) - if applied { - t.Fatal("a declaration signed by another key was applied") - } - if report.Refused != ErrForged.Error() { - t.Errorf("refused, but not as a forgery: %q", report.Refused) - } -} - -func TestATamperedDeclarationIsNeverApplied(t *testing.T) { - // A broker that changed the declaration in flight, keeping the signature. This is what makes - // pinning the transport insufficient on its own. - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - raw, err := json.Marshal(Signed{ - Declaration: []byte(`{"declaration":1,"resources":["something else entirely"]}`), - Signature: ed25519.Sign(private, []byte(`{"declaration":1}`)), - }) - if err != nil { - t.Fatal(err) - } - - report, applied := verified(t, public, raw) - if applied { - t.Fatal("a declaration altered after signing was applied") - } - if report.Refused != ErrForged.Error() { - t.Errorf("refused, but not as a forgery: %q", report.Refused) - } -} - -func TestAMalformedMessageIsToldApartFromAForgery(t *testing.T) { - // novox/hq ADR 0004 requires these to be distinguishable: one means somebody is trying, the - // other means something is broken, and they need different responses from a person. - public, _, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - report, applied := verified(t, public, []byte("this is not a message")) - if applied { - t.Fatal("something unparseable was applied") - } - if report.Refused == ErrForged.Error() { - t.Error("a malformed message was reported as a forgery; those must be distinguishable") - } -} - func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { // The contract with the control plane, which defines these separately. A matching test lives // there; rename a field on either side and both fail. @@ -117,7 +12,9 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { value any expect []string }{ - {Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}}, + // novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence. + {Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}}, + []string{"node", "writer", "epoch", "sequence"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, // novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what diff --git a/internal/link/movesecret.go b/internal/link/movesecret.go index fdfee75..95463bb 100644 --- a/internal/link/movesecret.go +++ b/internal/link/movesecret.go @@ -173,7 +173,7 @@ const VaultWithin = 15 * time.Second // AskVault asks one of the vault's verbs in this node's engine's own name over the link open now, and answers its // result, or its refusal as an error. -func (q *Queue) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) { +func (q *Session) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) { q.init() q.mu.Lock() bus := q.bus diff --git a/internal/link/movesecret_test.go b/internal/link/movesecret_test.go index 5cb244a..8457711 100644 --- a/internal/link/movesecret_test.go +++ b/internal/link/movesecret_test.go @@ -112,7 +112,7 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) { } defer vault.Unsubscribe() l := &natsLink{conn: conn, js: js, node: m.Node, signer: m.Signer} - q := &Queue{Membership: m} + q := &Session{Membership: m} q.attach(t.Context(), l) l.HearMoveSecrets(func(s MoveSecret) (MovedSecret, error) { raw, err := q.AskVault(t.Context(), "move-in", map[string]string{"module": s.Module}) @@ -135,7 +135,7 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) { if err := json.Unmarshal(reply.Data, &said); err != nil || said.Secret != "own/"+m.Node+"/postgres/superuser-password" { t.Fatalf("answered %s, %v", reply.Data, err) } - if _, err := (&Queue{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) { + if _, err := (&Session{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) { t.Fatalf("a queue with no link asked the vault: %v", err) } } diff --git a/internal/link/newest_test.go b/internal/link/newest_test.go deleted file mode 100644 index 5eb12a5..0000000 --- a/internal/link/newest_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package link - -import ( - "sync" - "testing" - "time" -) - -type said struct { - body []byte - - mu sync.Mutex - handled bool -} - -func (s *said) Body() []byte { return s.body } -func (s *said) Handled() error { - s.mu.Lock() - defer s.mu.Unlock() - s.handled = true - return nil -} -func (s *said) wasHandled() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.handled -} - -func arriving(bodies ...string) chan Declaration { - ch := make(chan Declaration, 8) - for _, b := range bodies { - ch <- &said{body: []byte(b)} - } - return ch -} - -// newest is what the queue's worker would take from what the link gathered behind `first`. -func newest(arriving <-chan Declaration, first Declaration, window time.Duration) (Declaration, []Declaration) { - return pick(gather(arriving, first, window)) -} - -// A machine asked to be five things becomes the last one: what is already waiting supersedes what -// arrived first, and everything set aside is named so it can be reported. -func TestWhatIsAlreadyWaitingSupersedesWhatArrivedFirst(t *testing.T) { - waiting := arriving("two", "three", "four") - apply, superseded := newest(waiting, &said{body: []byte("one")}, 50*time.Millisecond) - if string(apply.Body()) != "four" { - t.Fatalf("applied %q, not the newest", apply.Body()) - } - if len(superseded) != 3 || string(superseded[0].Body()) != "one" || - string(superseded[2].Body()) != "three" { - t.Fatalf("set aside %d: %v", len(superseded), superseded) - } -} - -// One declaration with nothing behind it is applied as it always was, after the window. -func TestALoneDeclarationIsAppliedAfterTheWindow(t *testing.T) { - began := time.Now() - apply, superseded := newest(arriving(), &said{body: []byte("only")}, 30*time.Millisecond) - if string(apply.Body()) != "only" || len(superseded) != 0 { - t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded)) - } - if time.Since(began) < 30*time.Millisecond { - t.Fatal("did not wait the window for a straggler") - } -} - -// A straggler within the window is taken; one after it is the next push. -func TestAStragglerWithinTheWindowIsTaken(t *testing.T) { - waiting := arriving() - go func() { - time.Sleep(20 * time.Millisecond) - waiting <- &said{body: []byte("late")} - }() - apply, superseded := newest(waiting, &said{body: []byte("first")}, 100*time.Millisecond) - if string(apply.Body()) != "late" || len(superseded) != 1 { - t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded)) - } -} diff --git a/internal/link/nothing_alive_test.go b/internal/link/nothing_alive_test.go index 300bea5..3945bb0 100644 --- a/internal/link/nothing_alive_test.go +++ b/internal/link/nothing_alive_test.go @@ -35,7 +35,7 @@ func (l *listeningForAlive) count() int { func TestTheLinkLoopPublishesNoAlive(t *testing.T) { m, _ := aMember(t) l := &listeningForAlive{quietLink: newQuietLink()} - q := aQueue(m, nil, &keptInMemory{}) + q := &Session{Membership: m, Timeout: time.Second} ctx, stop := context.WithCancel(context.Background()) done := make(chan error, 1) diff --git a/internal/link/order_test.go b/internal/link/order_test.go deleted file mode 100644 index 529d06c..0000000 --- a/internal/link/order_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package link - -import ( - "encoding/json" - "testing" - "time" -) - -// The drain picked the last to arrive. A backlog longer than the batch, or a broker that split a -// burst, delivered a superseded declaration last (novox/hq 04-ISSUES/107). - -func sequenced(t *testing.T, n int64) *said { - t.Helper() - inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{}, "sequence": n}) - if err != nil { - t.Fatal(err) - } - body, err := json.Marshal(Signed{Declaration: inner, Signature: []byte("s")}) - if err != nil { - t.Fatal(err) - } - return &said{body: body} -} - -func TestTheDrainKeepsTheHighestSequenceNotTheLastToArrive(t *testing.T) { - waiting := make(chan Declaration, 8) - waiting <- sequenced(t, 9) - waiting <- sequenced(t, 4) // arrived last, composed earlier - latest, aside := newest(waiting, sequenced(t, 8), 30*time.Millisecond) - if got := orderOf(latest.Body()).Sequence; got != 9 { - t.Fatalf("the drain kept sequence %d, and 9 was waiting", got) - } - if len(aside) != 2 { - t.Fatalf("%d set aside, wanted 2 (the 8 and the late 4)", len(aside)) - } -} - -func TestWithoutSequencesTheLastToArriveStillWins(t *testing.T) { - // The behaviour this had before, kept for a controller that sends no order. - apply, aside := newest(arriving("two", "three"), &said{body: []byte("one")}, 30*time.Millisecond) - if string(apply.Body()) != "three" || len(aside) != 2 { - t.Fatalf("applied %q with %d set aside", apply.Body(), len(aside)) - } -} - -func TestAnUnreadableBodyClaimsNoOrder(t *testing.T) { - if got := orderOf([]byte("not json")); got != (Order{}) { - t.Fatalf("garbage claimed %+v", got) - } -} diff --git a/internal/link/queue.go b/internal/link/queue.go deleted file mode 100644 index ddc2ea5..0000000 --- a/internal/link/queue.go +++ /dev/null @@ -1,647 +0,0 @@ -package link - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/health" -) - -// Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1). -// -// **One worker applies; everything else asks it to.** A declaration delivered over the link and the -// five-minute reconcile are two reasons to *enqueue* the same act, never two paths that apply. Before -// this they were two, each taking an apply lock, and every order between them was a fault met on the -// live mesh: the reconcile read what was kept before a delivery and applied it over the newer one -// (novox/hq issues 257, 261); the reconcile went first and its report about the older declaration -// reached the mesh after the delivery's (267). Each fix closed one order and left the class open. -// -// The queue holds at most one pending request, coalesced: whatever has been delivered and not yet -// taken, and whether a reconcile is due. When the worker starts it takes the **newest declaration held -// at that moment**, sets the rest aside — each reported as set aside, then settled unapplied — applies -// it once, and makes **one report**, naming the declaration's order and its own report sequence. A -// reconcile due while a delivery waits is that delivery's apply: applying the newest thing the mesh -// said is what reconciling is. Only with nothing delivered does a reconcile apply what was kept, read -// once its turn has come. -// -// **Reports leave in the order they are made**, because the worker makes and says them, one at a time. -// A report made while the link is down waits: an apply's in the unsaid store (issue 264), a -// reconcile's in one slot replaced by anything newer, and both are said, in that order, when the next -// link opens — before anything newly delivered is applied. -type Queue struct { - // Membership is whose declarations these are: their signer, and the node the reports are from. - Membership Membership - // Apply applies a declaration the mesh signed. - Apply Applier - // Reconcile holds the machine to what it was last told, read once it is this act's turn. False - // when there is nothing to hold it to, or the reconcile could not run; it says why itself. - Reconcile Reconcile - // News is whether a reconcile's report is worth saying unasked (novox/hq ADR 0100, ADR 0140). Nil - // is never: a reconcile is otherwise silent. - News func(Report) bool - // Heard is told every account of the machine the mesh has taken — an apply's or a reconcile's, - // not a refusal or a declaration set aside, which say nothing about the machine. Nil is allowed. - Heard func(Report) - // Unsaid keeps an apply's report until the mesh has taken it (issue 264). Nil keeps nothing. - Unsaid Unsaid - // Numbers keeps the report sequence and the refusals counted, across restarts and self-updates. - // Nil counts in memory, which is only right where nothing outlives the process — a test. - Numbers Numbers - // HandOver judges and records a directory used as found handed to the mesh by the operator, asked by - // the controller over the link (novox/hq issue 356). Nil answers every such ask with a refusal. - HandOver HandOverFunc - // SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the - // controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal. - SetuidSearch SetuidSearchFunc - // JudgeDirectories judges this node's declared directories read-only, asked by the controller over the - // link (novox/hq issue 358). Nil answers every such ask with a refusal. - JudgeDirectories JudgeDirectoriesFunc - // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved - // warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal. - AccessAct AccessActFunc - // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of - // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. - MoveSecret MoveSecretFunc - Say Announce - Timeout time.Duration - - once sync.Once - wake chan struct{} - - mu sync.Mutex - idle *sync.Cond - // delivering is a delivered declaration in hand: its report is owed on the link it came over, so - // that link is not let go until it is said (detach). A reconcile in hand holds no link — its report - // waits for the next one if this one goes. - delivering bool - waiting []Declaration // delivered and not yet taken, in the order they arrived - due bool // a reconcile asked for - // asked is the mesh asking this node to say what it last applied (the `report` verb, to-be 45 §6): - // the reconcile it enqueues says its report whether or not it is news. - asked bool - bus Bus // the link open now; nil while there is none - unasked *Report // a reconcile's report not yet said, made while there was no link - - // saying is held while a report is made and said, so the order they leave in is the order they - // are made in — whoever says them, the worker or a link opening. - saying sync.Mutex - counted bool - numbers numbered - - // haves is what this node-engine writes beside its reports (novox/hq ADR 0429 §4, issue 1132). - haves haveState -} - -// Reconcile is the reconcile's act, run by the queue's worker. See Queue.Reconcile. -type Reconcile func(ctx context.Context) (Report, bool) - -// Numbers is where the queue keeps what it counts, so a successor goes on from where it stopped. -type Numbers interface { - // Read is what was kept: zero for a node that never reported, an error when it cannot be read — - // never zero for "could not tell". - Read() (reportSequence, refusedOlder int64, err error) - // Save keeps them, on disk when it returns. - Save(reportSequence, refusedOlder int64) error -} - -type numbered struct{ sequence, refused int64 } - -func (q *Queue) init() { - q.once.Do(func() { - q.wake = make(chan struct{}, 1) - q.idle = sync.NewCond(&q.mu) - if q.Say == nil { - q.Say = func(string) {} - } - }) -} - -// Deliver enqueues what the link delivered, in the order it arrived. It never waits for an apply. -func (q *Queue) Deliver(arrived ...Declaration) { - q.init() - q.mu.Lock() - q.waiting = append(q.waiting, arrived...) - q.mu.Unlock() - q.signal() -} - -// ReconcileDue enqueues a reconcile. One asked for while another is waiting is the same one. -func (q *Queue) ReconcileDue() { - q.init() - q.mu.Lock() - q.due = true - q.mu.Unlock() - q.signal() -} - -// ReportAsked is the mesh asking this node to say again what it last applied (novox/hq to-be 45 §6, the -// `report` verb its healer H1 asks when a send went unreported). **An enqueue, like everything else**: a -// reconcile, whose report — the account of the declaration this node keeps, the last it applied — is -// said whether or not it is news. A delivery waiting meanwhile is applied instead and reported, which -// answers the question better. Asked again before the worker takes it is the same question. -func (q *Queue) ReportAsked() { - q.init() - q.mu.Lock() - q.due, q.asked = true, true - q.mu.Unlock() - q.signal() -} - -func (q *Queue) signal() { - select { - case q.wake <- struct{}{}: - default: - // One is already waiting to be read, and the worker takes everything pending when it does. - } -} - -// Run is the worker. It returns once the context ends and the act in hand is finished and said: -// asked to stop — the host standing aside for its successor — nothing further is applied. -func (q *Queue) Run(ctx context.Context) { - q.init() - for { - for q.step(ctx) { - } - select { - case <-ctx.Done(): - return - case <-q.wake: - } - } -} - -// step takes what is pending and does it, once. False when there was nothing to do, or the queue -// was asked to stop. -func (q *Queue) step(ctx context.Context) bool { - q.mu.Lock() - if ctx.Err() != nil || (len(q.waiting) == 0 && !q.due) { - q.mu.Unlock() - return false - } - batch, due, asked := q.waiting, q.due, q.asked - q.waiting, q.due, q.asked = nil, false, false - q.delivering = len(batch) > 0 - q.mu.Unlock() - defer func() { - q.mu.Lock() - q.delivering = false - q.idle.Broadcast() - q.mu.Unlock() - }() - - if len(batch) == 0 { - q.reconcile(ctx, asked) - return true - } - - latest, aside := pick(batch) - for _, old := range aside { - if d := declaredIn(old.Body()); d != "" && d == declaredIn(latest.Body()) { - // The same declaration again — delivered twice while the worker was busy. Nothing is set - // aside: it is about to be applied. - _ = old.Handled() - continue - } - q.Say(fmt.Sprintf("set aside declaration %s (%s): a newer one arrived with it", - short(declaredIn(old.Body())), orderOf(old.Body()).Words())) - q.tell(ctx, Report{Declared: declaredIn(old.Body()), Order: orderOf(old.Body()), - Superseded: declaredIn(latest.Body())}, setAside, old) - } - - report := handleBody(q.withFiles(ctx), q.Membership, latest.Body(), q.Apply) - switch { - case report.OlderThan != nil: - // **Refused, counted and said** (rule 2): one line here, the count on this and every later - // report, and the report itself naming what was refused and what is held. - total := q.countRefusal() - q.Say(fmt.Sprintf("refused declaration %s (%s): this node applied %s, from a newer lease "+ - "holder — %d refused as older so far", short(report.Declared), report.Order.Words(), - report.OlderThan.Words(), total)) - case report.OlderGeneration != nil: - // Said here, and the report names it, so the controller raises it naming the sender (novox/hq - // issue 234). Not counted with the lease's refusals: those are S13's, by epoch. - q.Say(fmt.Sprintf("refused declaration %s (%s): it was composed from assignment generation %d, and "+ - "this node applied generation %d", short(report.Declared), report.Order.Words(), - report.OlderGeneration.Generation, report.OlderGeneration.Applied)) - case report.Refused != "": - q.Say("refused a declaration: " + report.Refused) - case len(report.Failed) > 0: - q.Say(fmt.Sprintf("applied %d and failed: %v%s", - len(report.Applied), report.Failed, heldNote(report.Held))) - default: - q.Say(fmt.Sprintf("applied %d resource(s)%s", len(report.Applied), heldNote(report.Held))) - } - if due && report.Refused != "" { - // Nothing was applied, so the reconcile this stood in for has not happened. It runs next. - q.mu.Lock() - q.due = true - q.mu.Unlock() - } - // A refusal as older is not kept to be said again: what the mesh waits for from this node is the - // account of the last apply, and that is still what is kept. - kind := applied - if report.OlderThan != nil || report.OlderGeneration != nil { - kind = refusedOlder - } - q.tell(ctx, report, kind, latest) - return true -} - -func (q *Queue) reconcile(ctx context.Context, asked bool) { - if q.Reconcile == nil { - if asked { - q.Say("asked to say what this node last applied, and it holds nothing to reconcile with") - } - return - } - report, ok := q.Reconcile(q.withFiles(ctx)) - if !ok { - if asked { - q.Say("asked to say what this node last applied, and the reconcile could not say it") - } - return - } - if !asked && (q.News == nil || !q.News(report)) { - // Not worth a report, but a reconcile found how each module is now: the have keys say it (issue 1132), so a - // node-engine that restarted writes them again without waiting for a change. - if q.noteApply(report) { - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - q.writeHaves(ctx, bus) - } - return - } - if asked { - q.Say(fmt.Sprintf("asked by the mesh, saying again what this node last applied: declaration %s (%s)", - short(report.Declared), report.Order.Words())) - } - q.tell(ctx, report, unasked) -} - -// What a report is, for what tell does with it besides saying it. -type kindOf int - -const ( - // applied is an apply's account: kept until said (issue 264), and fresher than any reconcile's - // report still waiting. - applied kindOf = iota - // setAside is a declaration a newer one took the place of, unapplied. - setAside - // refusedOlder is a declaration refused as older than what was applied. Not kept: what the mesh - // waits for from this node is still the account of the last apply. - refusedOlder - // unasked is a reconcile's report. Not said — no link, or the bus refused it — the newest waits - // for the next link, and anything the worker says before then replaces it. - unasked -) - -// tell makes a report this node's — its node, its report sequence, the refusals counted — and says it -// on the link open now. Kept first, when it is an apply's, so a report lost between here and the bus -// is said by whichever host links next. Each declaration it settles is settled after it is said, -// either way. True when the broker took it. -func (q *Queue) tell(ctx context.Context, r Report, kind kindOf, settle ...Declaration) bool { - q.saying.Lock() - defer q.saying.Unlock() - - r = q.stamp(r) - keep := kind == applied - if keep { - // What this says supersedes any reconcile's report still waiting to be said. - q.unasked = nil - if q.Unsaid != nil { - if err := q.Unsaid.Keep(r); err != nil { - q.Say("cannot keep this apply's report until it is said: " + err.Error()) - } - } - } - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - - published := false - if bus != nil { - // **Said even when the link is being let go** (issue 264): not cancelled with it, still - // bounded by the timeout, and the link is not closed until this returns (detach). - published = publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) - } - if !published && kind == unasked { - q.unasked = &r - } - if published { - // After the report, which is sent first and unchanged; a write the bucket refuses never undoes it (issue 1132). - if q.noteApply(r) { - q.writeHaves(ctx, bus) - } - if keep && q.Unsaid != nil { - if err := q.Unsaid.Said(r.Declared); err != nil { - q.Say("said this apply's report, and cannot forget it: " + err.Error()) - } - } - if q.Heard != nil && (kind == applied || kind == unasked) && r.Refused == "" { - q.Heard(r) - } - } - // Settled after the report is published. A node that dies between applying and reporting - // leaves the declaration with the mesh and applies it again on return, which is safe because - // applying is reconciliation — it converges rather than repeating. - for _, d := range settle { - _ = d.Handled() - } - return published -} - -// stamp gives a report its node, the next report sequence and the refusals counted. Called holding -// saying. -func (q *Queue) stamp(r Report) Report { - q.load() - q.numbers.sequence++ - q.keepNumbers() - r.Node = q.Membership.Node - r.ReportSequence = q.numbers.sequence - r.RefusedOlder = q.numbers.refused - // Every report of a node-engine that orders its reports says it reads a generation (novox/hq issue - // 234): this build does, and the controller sends one only to a machine that said so. - r.ReadsGeneration = true - // And the module of each resource and the build of each module (novox/hq ADR 0429 §4, issue 1132), the same way. - r.ReadsModules = true - return r -} - -func (q *Queue) countRefusal() int64 { - q.saying.Lock() - defer q.saying.Unlock() - q.load() - q.numbers.refused++ - q.keepNumbers() - return q.numbers.refused -} - -// load reads what was counted, once. **Unreadable is said, and the sequence goes on from above -// anything it can have reached** — the time in milliseconds — rather than from one, which would make -// every report this host says next older than what it already said, and refused. -func (q *Queue) load() { - if q.counted { - return - } - q.counted = true - if q.Numbers == nil { - return - } - sequence, refused, err := q.Numbers.Read() - if err != nil { - floor := time.Now().UnixMilli() - q.Say(fmt.Sprintf("cannot read this node's report sequence: %v — going on from %d, above any "+ - "it can have reached, and counting refusals again from none", err, floor)) - q.numbers = numbered{sequence: floor} - return - } - q.numbers = numbered{sequence: sequence, refused: refused} -} - -func (q *Queue) keepNumbers() { - if q.Numbers == nil { - return - } - if err := q.Numbers.Save(q.numbers.sequence, q.numbers.refused); err != nil { - // Said, and not fatal: the report is still worth saying. What is lost is that a successor - // could number one again. - q.Say("cannot keep this node's report sequence: " + err.Error()) - } -} - -// attach is a link opening. **What the last apply did, if the mesh never heard it**, is said first, -// exactly as it was made; then a reconcile's report that waited; and only then may the worker say -// anything on it — so a report said again can never land after, and read as newer than, a later one. -func (q *Queue) attach(ctx context.Context, bus Bus) { - q.init() - q.saying.Lock() - defer q.saying.Unlock() - reporting := context.WithoutCancel(ctx) - q.load() - - said := true - if q.Unsaid != nil { - switch kept, ok, err := q.Unsaid.Pending(); { - case err != nil: - q.Say("cannot read the report this node kept unsaid: " + err.Error()) - case ok: - q.Say("saying again what the last apply did: its report never reached the mesh") - if said = publishReport(reporting, bus, q.Membership, kept, q.Say, q.Timeout); said { - if err := q.Unsaid.Said(kept.Declared); err != nil { - q.Say("said the kept report, and cannot forget it: " + err.Error()) - } - if q.Heard != nil && kept.Refused == "" { - q.Heard(kept) - } - } - } - } - if said && q.unasked != nil { - if publishReport(reporting, bus, q.Membership, *q.unasked, q.Say, q.Timeout) { - if q.Heard != nil { - q.Heard(*q.unasked) - } - q.unasked = nil - } - } - - q.mu.Lock() - q.bus = bus - q.mu.Unlock() -} - -// withFiles is ctx carrying the link now open, so an apply can fetch a declaration's files given by digest (novox/hq ADR -// 0486, issue 1695). A queue with no link open carries none. -func (q *Queue) withFiles(ctx context.Context) context.Context { - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return ctx - } - return WithFiles(ctx, bus) -} - -// detach is the link closing: it waits for a delivery in hand to be applied and said — an apply that -// stood aside for its successor is reported on this link before it is let go (issue 264) — and then -// nothing more is said on it. A reconcile in hand is not waited for: a link that was dropped or roused -// is opened again at once, and the reconcile's report, if it is news, waits for it. -func (q *Queue) detach(bus Bus) { - q.init() - q.mu.Lock() - for q.delivering { - q.idle.Wait() - } - q.mu.Unlock() - - q.saying.Lock() - defer q.saying.Unlock() - q.mu.Lock() - if q.bus == bus { - q.bus = nil - } - q.mu.Unlock() -} - -// pick is the newest of what was delivered and the rest, in arrival order, to set aside: by order -// when the declarations claim one, by arrival when they do not (Order.Supersedes). -func pick(batch []Declaration) (Declaration, []Declaration) { - latest := batch[0] - var aside []Declaration - for _, next := range batch[1:] { - if orderOf(next.Body()).Supersedes(orderOf(latest.Body())) { - aside = append(aside, latest) - latest = next - continue - } - aside = append(aside, next) - } - return latest, aside -} - -// orderOf is the order a signed declaration claims, or none when it claims none or cannot be read. -// Read from the envelope alone; the signature is verified when the winner is applied, and a forged -// message that lied about its order would only set aside real ones — which are reported as set aside, -// and the next push sends the current one again. -func orderOf(body []byte) Order { - var signed Signed - if err := json.Unmarshal(body, &signed); err != nil { - return Order{} - } - var o Order - if err := json.Unmarshal(signed.Declaration, &o); err != nil || o.Epoch < 0 || o.Sequence < 0 { - return Order{} - } - return o -} - -// declaredIn names a signed declaration as the mesh does — the digest of the declaration's bytes, the -// same the apply's report carries — for a report about one that was not applied. Empty if the message -// is not one: a forged or garbled message is refused when its turn comes; here it is only named. -func declaredIn(body []byte) string { - var signed Signed - if err := json.Unmarshal(body, &signed); err != nil || len(signed.Declaration) == 0 { - return "" - } - sum := sha256.Sum256(signed.Declaration) - return hex.EncodeToString(sum[:]) -} - -// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the -// machine's networking its engine judges). -func (q *Queue) Linked() bool { - q.init() - q.mu.Lock() - defer q.mu.Unlock() - return q.bus != nil -} - -// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether -// the bus took it. **Not through the worker**: a statement is a word about the machine as it is, not an -// account of an apply, and it must not wait behind one — a container crash-looping while a long apply -// runs is said while it runs. False with no link open, or one that cannot say it; the caller says it -// again on its next look, and the next report carries it whatever happens. -func (q *Queue) SayHealth(ctx context.Context, h Health) bool { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - sayer, ok := bus.(HealthBus) - if bus == nil || !ok { - return false - } - body, err := json.Marshal(HealthSaid{Node: q.Membership.Node, Health: h}) - if err != nil { - return false - } - saying, cancel := context.WithTimeout(ctx, q.timeout()) - defer cancel() - if err := sayer.Health(saying, q.Membership.Node, body); err != nil { - q.Say("could not say this machine's health: " + err.Error()) - return false - } - // A look that changed a module's health changes its have (issue 1132). - if q.noteHealth(h) { - q.writeHaves(ctx, bus) - } - return true -} - -func (q *Queue) timeout() time.Duration { - if q.Timeout <= 0 { - return 10 * time.Second - } - return q.Timeout -} - -// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR -// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open -// or nothing answered — said as the check's failure, never as healthy. -func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return health.Answer{}, errors.New("no link to the bus is open") - } - asker, ok := bus.(ToolBus) - if !ok { - return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool") - } - reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}")) - if err != nil { - return health.Answer{}, err - } - return ReadToolAnswer(reply) -} - -// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over -// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for. -func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error) { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) - } - asker, ok := bus.(ToolBus) - if !ok { - return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked) - } - return asker.Ask(ctx, CLISubject(q.Membership.Node), body) -} - -// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any -// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272). -var ErrNotAsked = errors.New("not asked") - -// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call — -// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the -// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the -// SDK cannot check is unhealthy too, so a malformed one never excuses anything. -func ReadToolAnswer(reply []byte) (health.Answer, error) { - var env struct { - Result json.RawMessage `json:"result"` - Error string `json:"error"` - } - if err := json.Unmarshal(reply, &env); err != nil { - return health.Answer{}, fmt.Errorf("its answer is not one: %w", err) - } - if env.Error != "" { - return health.Unhealthy(env.Error), nil - } - a, err := health.Read(env.Result) - if err != nil { - return health.Unhealthy("it answered something other than healthy or not"), nil - } - return a, nil -} diff --git a/internal/link/queue_test.go b/internal/link/queue_test.go deleted file mode 100644 index 3e0e7b2..0000000 --- a/internal/link/queue_test.go +++ /dev/null @@ -1,588 +0,0 @@ -package link - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "errors" - "reflect" - "strings" - "sync" - "testing" - "time" -) - -// The node-engine's one apply queue (novox/hq to-be 45 §6): a delivery and the reconcile are two -// reasons to enqueue the same act, the newest declaration is applied once, and the reports leave in -// the order they are made. These are the races of issues 257, 261, 264 and 267 replayed against it. - -func aQueue(m Membership, apply Applier, kept Unsaid) *Queue { - return &Queue{Membership: m, Apply: apply, Unsaid: kept, Timeout: time.Second} -} - -// runWorker runs the queue's worker until ctx ends; the channel closes when it has. -func runWorker(ctx context.Context, q *Queue) <-chan struct{} { - done := make(chan struct{}) - go func() { - defer close(done) - q.Run(ctx) - }() - return done -} - -// ordered is a signed declaration claiming an order, with a resource id so two are distinct bytes. -func ordered(t *testing.T, key ed25519.PrivateKey, id string, epoch, sequence int64) *said { - t.Helper() - inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{map[string]any{"id": id}}, - "epoch": epoch, "sequence": sequence}) - if err != nil { - t.Fatal(err) - } - return &said{body: signedBy(t, key, inner)} -} - -// applies records what the worker applied, by the id of the declaration's first resource, and -// reports it as applied — with the order the declaration carried, as the host's own apply does. -type applies struct { - mu sync.Mutex - seen []string -} - -func (a *applies) apply(_ context.Context, raw, _ []byte) Report { - var d struct { - Resources []struct { - ID string `json:"id"` - } `json:"resources"` - Order - } - _ = json.Unmarshal(raw, &d) - id := "" - if len(d.Resources) > 0 { - id = d.Resources[0].ID - } - a.mu.Lock() - a.seen = append(a.seen, id) - a.mu.Unlock() - return Report{Declared: digest(raw), Order: d.Order, Applied: []string{id}} -} - -func (a *applies) all() []string { - a.mu.Lock() - defer a.mu.Unlock() - return append([]string(nil), a.seen...) -} - -func digest(raw []byte) string { - b, _ := json.Marshal(Signed{Declaration: raw}) - return declaredIn(b) -} - -// eventually waits for a condition the worker reaches on its own time. -func eventually(t *testing.T, what string, ok func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for !ok() { - if time.Now().After(deadline) { - t.Fatal(what) - } - time.Sleep(5 * time.Millisecond) - } -} - -// accounts is the reports the mesh heard that are an apply's account, not a set-aside's. -func accounts(reports []Report) []Report { - var out []Report - for _, r := range reports { - if r.Superseded == "" { - out = append(out, r) - } - } - return out -} - -// **R2: a reconcile due during a delivery** (issues 257, 261, 267). The reconcile is asked for while a -// delivery waits: the worker applies the delivery once — applying the newest thing the mesh said is -// what reconciling is — and makes one report, naming the newest sequence. The reconcile never applies -// what was kept before. -func TestAReconcileDueDuringADeliveryIsThatDeliverysApply(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - reconciled := 0 - q := aQueue(m, a.apply, &keptInMemory{}) - q.Reconcile = func(context.Context) (Report, bool) { - reconciled++ - return Report{Declared: "the one kept before", Applied: []string{"old"}}, true - } - q.News = func(Report) bool { return true } - q.attach(context.Background(), l) - - q.Deliver(ordered(t, key, "new", 7, 12)) - q.ReconcileDue() - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"new"}) { - t.Fatalf("applied %v; one apply of the newest was wanted", got) - } - if reconciled != 0 { - t.Fatalf("the reconcile applied what was kept %d time(s) beside the delivery", reconciled) - } - r := l.said()[0] - if r.Sequence != 12 || r.Epoch != 7 || r.ReportSequence != 1 || r.Node != m.Node { - t.Fatalf("the report does not name the order it applied: %+v", r) - } -} - -// **The other order** (issue 267): the reconcile is running when a delivery arrives. Its report, about -// the declaration kept then, leaves first; the delivery's leaves after it, with a higher report -// sequence. The mesh's latest word is the newest declaration, whichever arrived first. -func TestAReconcileRunningWhenADeliveryArrivesIsSaidBeforeIt(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - inReconcile, release := make(chan struct{}), make(chan struct{}) - q.Reconcile = func(context.Context) (Report, bool) { - close(inReconcile) - <-release - return Report{Declared: "d1", Order: Order{Epoch: 7, Sequence: 11}, Applied: []string{"a"}, - Outward: []string{"eth0"}}, true - } - q.News = func(Report) bool { return true } - q.attach(context.Background(), l) - - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.ReconcileDue() - <-inReconcile - q.Deliver(ordered(t, key, "b", 7, 12)) - close(release) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 2 }) - stop() - <-worker - - reports := l.said() - if reports[0].Declared != "d1" || reports[1].Sequence != 12 { - t.Fatalf("the reports left out of the order they were made: %+v", reports) - } - if reports[0].ReportSequence >= reports[1].ReportSequence { - t.Fatalf("the later report carries no higher report sequence: %d then %d", - reports[0].ReportSequence, reports[1].ReportSequence) - } -} - -// **Two deliveries in quick succession**: only the newest is applied, and there is one account of an -// apply — the first is reported as set aside, naming the one that took its place. -func TestTwoDeliveriesInQuickSuccessionApplyOnlyTheNewest(t *testing.T) { - m, key := aMember(t) - first, second := ordered(t, key, "first", 7, 12), ordered(t, key, "second", 7, 13) - l := newQuietLink(first, second) - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - - ctx, stop := context.WithCancel(context.Background()) - defer stop() - worker := runWorker(ctx, q) - go func() { _ = serve(ctx, l, m, q, nil, time.Second) }() - eventually(t, "nothing was reported", func() bool { return len(accounts(l.said())) == 1 }) - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"second"}) { - t.Fatalf("applied %v; only the newest was wanted", got) - } - reports := l.said() - if len(reports) != 2 || reports[0].Superseded != declaredIn(second.body) || - reports[0].Declared != declaredIn(first.body) || reports[0].Sequence != 12 { - t.Fatalf("the first was not reported as set aside for the second: %+v", reports) - } - if !first.wasHandled() || !second.wasHandled() { - t.Fatal("a declaration was left unsettled") - } -} - -// Deliveries that arrive while the worker is applying are coalesced: when it is free it takes the -// newest held at that moment — by order, not by arrival — and applies it once. -func TestDeliveriesWhileTheWorkerIsBusyAreOneApplyOfTheNewest(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - inApply, release := make(chan struct{}, 1), make(chan struct{}) - q := aQueue(m, func(ctx context.Context, raw, sig []byte) Report { - r := a.apply(ctx, raw, sig) - if r.Sequence == 1 { - inApply <- struct{}{} - <-release - } - return r - }, &keptInMemory{}) - q.attach(context.Background(), l) - - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.Deliver(ordered(t, key, "one", 7, 1)) - <-inApply - q.Deliver(ordered(t, key, "three", 7, 3)) - q.Deliver(ordered(t, key, "two", 7, 2)) // arrived last, composed earlier - q.ReconcileDue() // and the five-minute timer fired too - close(release) - eventually(t, "the newest was never reported", func() bool { return len(accounts(l.said())) == 2 }) - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"one", "three"}) { - t.Fatalf("applied %v; the one in hand and then only the newest were wanted", got) - } - last := accounts(l.said())[1] - if last.Sequence != 3 { - t.Fatalf("the last account names sequence %d, not the newest", last.Sequence) - } -} - -// **An older epoch is refused, counted and said** (rule 2): the refusal's report names the refused -// declaration and what this node holds, carries the count, and does not replace the kept account of -// the last apply — which is what the mesh is waiting for from this node. -func TestADeclarationFromAnOlderEpochIsRefusedCountedAndSaid(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - kept := &keptInMemory{} - lastApply := Report{Declared: "d-applied", Order: Order{Epoch: 57, Sequence: 3}, Applied: []string{"a"}} - _ = kept.Keep(lastApply) - stale := ordered(t, key, "stale", 41, 12) - q := aQueue(m, func(_ context.Context, raw, _ []byte) Report { - // What the host's apply answers for a declaration older than the one it kept. - return Report{Declared: digest(raw), Order: Order{Epoch: 41, Sequence: 12}, - OlderThan: &Order{Epoch: 57, Sequence: 3}, Refused: "older than what this node applied"} - }, kept) - var lines saidSoFar - q.Say = lines.say - l.refusing = true // nothing is said on linking: the kept account stays kept - q.attach(context.Background(), l) - l.mu.Lock() - l.refusing = false - l.mu.Unlock() - - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.Deliver(stale) - eventually(t, "the refusal was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - - r := l.said()[0] - if r.OlderThan == nil || *r.OlderThan != (Order{Epoch: 57, Sequence: 3}) || r.Epoch != 41 || - r.Declared != declaredIn(stale.body) || r.RefusedOlder != 1 { - t.Fatalf("the refusal does not name what was refused, what is held, and the count: %+v", r) - } - if !strings.Contains(lines.all(), "1 refused as older so far") { - t.Fatalf("the refusal was not said in the log:\n%s", lines.all()) - } - if still, ok, _ := kept.Pending(); !ok || still.Declared != "d-applied" { - t.Fatalf("the refusal replaced the kept account of the last apply: %+v", still) - } - if !stale.wasHandled() { - t.Fatal("the refused declaration was not settled: the mesh would deliver it again") - } -} - -// **Restart with an unsaid report**: the next host says it first, exactly as it was made, and numbers -// what it says next above it — the report sequence goes on across the restart rather than from one. -func TestAfterARestartTheUnsaidReportIsSaidFirstAndTheNumbersGoOn(t *testing.T) { - m, key := aMember(t) - kept := &keptInMemory{} - numbers := &numbersInMemory{} - - // The first host applies; its report never reaches the mesh, and it is gone. - first := newQuietLink() - first.refusing = true - q1 := aQueue(m, (&applies{}).apply, kept) - q1.Numbers = numbers - q1.attach(context.Background(), first) - ctx1, stop1 := context.WithCancel(context.Background()) - w1 := runWorker(ctx1, q1) - q1.Deliver(ordered(t, key, "a", 7, 12)) - eventually(t, "the first host never kept its report", func() bool { _, ok, _ := kept.Pending(); return ok }) - stop1() - <-w1 - lost, _, _ := kept.Pending() - - // The next host links, and is then delivered something new. - next := newQuietLink() - q2 := aQueue(m, (&applies{}).apply, kept) - q2.Numbers = numbers - q2.attach(context.Background(), next) - ctx2, stop2 := context.WithCancel(context.Background()) - w2 := runWorker(ctx2, q2) - q2.Deliver(ordered(t, key, "b", 7, 13)) - eventually(t, "the next host's apply was never reported", func() bool { return len(next.said()) == 2 }) - stop2() - <-w2 - - reports := next.said() - if !reflect.DeepEqual(reports[0], lost) { - t.Fatalf("the unsaid report was not said first, as it was made: %+v", reports[0]) - } - if reports[1].Sequence != 13 || reports[1].ReportSequence <= lost.ReportSequence { - t.Fatalf("the next host numbered from one again: %d after %d", - reports[1].ReportSequence, lost.ReportSequence) - } -} - -// A report sequence that cannot be read is said, and numbering goes on above anything it can have -// reached — never from one, which would make every report older than those already said. -func TestUnreadableNumbersAreSaidAndNumberingGoesOnAboveThem(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - q := aQueue(m, nil, nil) - q.Numbers = &numbersInMemory{broken: errors.New("unreadable")} - var lines saidSoFar - q.Say = lines.say - q.News = func(Report) bool { return true } - q.Reconcile = func(context.Context) (Report, bool) { return Report{Declared: "d"}, true } - before := time.Now().UnixMilli() - q.attach(context.Background(), l) - - q.ReconcileDue() - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "nothing was reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - if got := l.said()[0].ReportSequence; got < before { - t.Fatalf("numbering went on from %d, not above anything it can have reached", got) - } - if !strings.Contains(lines.all(), "cannot read this node's report sequence") { - t.Fatalf("the unreadable sequence was not said:\n%s", lines.all()) - } -} - -// A reconcile's report made with no link waits for the next one; an apply made before then replaces -// it, because the apply's account is fresher. -func TestAReconcileReportMadeOfflineWaitsAndIsReplacedByAnApply(t *testing.T) { - m, key := aMember(t) - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) - q.News = func(Report) bool { return true } - q.Reconcile = func(context.Context) (Report, bool) { - return Report{Declared: "d1", Outward: []string{"eth0"}}, true - } - - ctx, stop := context.WithCancel(context.Background()) - defer stop() - runWorker(ctx, q) - q.ReconcileDue() - eventually(t, "the reconcile's report was not held", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked != nil - }) - l := newQuietLink() - q.attach(context.Background(), l) - if reports := l.said(); len(reports) != 1 || reports[0].Declared != "d1" { - t.Fatalf("the reconcile's report was not said when the link opened: %+v", reports) - } - - // Offline again: a reconcile's report, then an apply of a delivery still in hand. - q.detach(l) - q.ReconcileDue() - eventually(t, "the second reconcile's report was not held", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked != nil - }) - q.Deliver(ordered(t, key, "x", 7, 2)) - eventually(t, "the apply did not replace the reconcile's report", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked == nil - }) -} - -// The contract with the controller, on the wire: the keys a declaration's order is read from, and the -// keys a report carries back. A rename here must break this test before it breaks a machine. -func TestTheOrderOnTheWire(t *testing.T) { - body, err := json.Marshal(Report{Node: "n", Declared: "d", Order: Order{Epoch: 57, Sequence: 3}, - ReportSequence: 9, OlderThan: &Order{Epoch: 58, Sequence: 1}, RefusedOlder: 2}) - if err != nil { - t.Fatal(err) - } - var wire map[string]any - _ = json.Unmarshal(body, &wire) - want := map[string]any{"node": "n", "declared": "d", "epoch": 57.0, "sequence": 3.0, - "report_sequence": 9.0, "older_than": map[string]any{"epoch": 58.0, "sequence": 1.0}, - "refused_older": 2.0} - if !reflect.DeepEqual(wire, want) { - t.Fatalf("a report's order on the wire is\n%s\nnot the contract", body) - } - - // An older host's report, and an older controller's declaration, claim no order. - if body, _ := json.Marshal(Report{Node: "n", Declared: "d"}); strings.Contains(string(body), "sequence") || - strings.Contains(string(body), "epoch") { - t.Fatalf("a report about a declaration with no order claims one: %s", body) - } - _, key := aMember(t) - if got := orderOf(signedBy(t, key, []byte(`{"declaration":1,"epoch":57,"sequence":3}`))); got != (Order{Epoch: 57, Sequence: 3}) { - t.Fatalf("a declaration's order was read as %+v", got) - } -} - -func TestOlder(t *testing.T) { - for _, c := range []struct { - in, held Order - older bool - }{ - {Order{41, 12}, Order{57, 3}, true}, // an older lease holder, whatever its sequence - {Order{57, 2}, Order{57, 3}, true}, // same holder, lower sequence - {Order{57, 3}, Order{57, 3}, false}, // the same declaration again: reconciling - {Order{58, 1}, Order{57, 3}, false}, // a new lease holder - {Order{0, 1}, Order{57, 3}, false}, // an older controller, or one rolled back: today's behaviour - {Order{41, 1}, Order{0, 9}, false}, // nothing held claims an epoch - {Order{57, 0}, Order{57, 3}, false}, // no sequence to compare - {Order{0, 0}, Order{0, 0}, false}, // neither claims anything - {Order{-1, 0}, Order{57, 3}, false}, // never a claim - {Order{57, 4}, Order{57, 3}, false}, // newer - {Order{56, 99}, Order{57, 1}, true}, // a higher sequence is no excuse for an older epoch - {Order{57, 1}, Order{56, 99}, false}, // nor a lower one a reason to refuse a newer epoch - {Order{57, 3}, Order{57, 0}, false}, // held with no sequence - {Order{100, 0}, Order{57, 0}, false}, // newer epoch, no sequences - {Order{10, 0}, Order{57, 0}, true}, // older epoch, no sequences - {Order{57, 2}, Order{0, 0}, false}, // nothing held at all - {Order{0, 2}, Order{0, 3}, false}, // sequence alone does not refuse on the link (no epoch) - {Order{57, 2}, Order{57, -1}, false}, // a held sequence below zero is not one - {Order{57, -1}, Order{57, 2}, false}, // nor an arriving one - {Order{-5, -5}, Order{-1, -1}, false}, // nothing below zero is an order - } { - if got := c.in.Older(c.held); got != c.older { - t.Errorf("%+v older than %+v = %v, want %v", c.in, c.held, got, c.older) - } - } -} - -func TestSupersedes(t *testing.T) { - for _, c := range []struct { - next, before Order - want bool - }{ - {Order{58, 1}, Order{57, 9}, true}, // a new lease holder - {Order{57, 9}, Order{58, 1}, false}, // the stale one arriving late - {Order{57, 4}, Order{57, 3}, true}, - {Order{57, 2}, Order{57, 3}, false}, // arrived last, composed earlier - {Order{0, 4}, Order{0, 3}, true}, - {Order{0, 2}, Order{0, 3}, false}, - {Order{}, Order{0, 3}, true}, // no order: by arrival - {Order{0, 3}, Order{}, true}, - } { - if got := c.next.Supersedes(c.before); got != c.want { - t.Errorf("%+v supersedes %+v = %v, want %v", c.next, c.before, got, c.want) - } - } -} - -// numbersInMemory is Numbers without a disk, surviving a "restart" by being shared. -type numbersInMemory struct { - mu sync.Mutex - sequence, refused int64 - broken error -} - -func (n *numbersInMemory) Read() (int64, int64, error) { - n.mu.Lock() - defer n.mu.Unlock() - return n.sequence, n.refused, n.broken -} - -func (n *numbersInMemory) Save(sequence, refused int64) error { - n.mu.Lock() - defer n.mu.Unlock() - n.sequence, n.refused = sequence, refused - return nil -} - -// A link dropped or roused while a reconcile is in hand is let go at once and opened again: the -// reconcile holds no link, and its report, if it is news, waits for the next one. Only a delivery in -// hand keeps its link until its report is said. -func TestALinkIsNotHeldForAReconcileInHand(t *testing.T) { - m, _ := aMember(t) - inReconcile, release := make(chan struct{}), make(chan struct{}) - q := aQueue(m, nil, nil) - q.News = func(Report) bool { return true } - q.Reconcile = func(context.Context) (Report, bool) { - close(inReconcile) - <-release - return Report{Declared: "d1", Outward: []string{"eth0"}}, true - } - l := newQuietLink() - q.attach(context.Background(), l) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.ReconcileDue() - <-inReconcile - - detached := make(chan struct{}) - go func() { q.detach(l); close(detached) }() - select { - case <-detached: - case <-time.After(2 * time.Second): - t.Fatal("the link was held while a reconcile ran") - } - close(release) - eventually(t, "the reconcile's report was not held for the next link", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked != nil - }) - stop() - <-worker - if len(l.said()) != 0 { - t.Fatalf("a report was said on a link let go: %+v", l.said()) - } -} - -// **Every report says this node-engine reads a declaration's generation** (novox/hq issue 234): the -// controller sends one only to a machine that said so, because an older node-engine refuses the key. -func TestEveryReportSaysItReadsAGeneration(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - q.attach(context.Background(), l) - q.Deliver(ordered(t, key, "new", 7, 12)) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - raw, err := json.Marshal(l.said()[0]) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(raw), `"reads_generation":true`) { - t.Fatalf("the report does not say it reads a generation: %s", raw) - } -} - -// 1132/AC2: every report says this node-engine reads a resource's module and a declaration's builds, beside the -// generation: the controller sends them only to a machine that said so, because an older node-engine refuses the keys. -func TestEveryReportSaysItReadsModules(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - q.attach(context.Background(), l) - q.Deliver(ordered(t, key, "new", 7, 12)) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - raw, err := json.Marshal(l.said()[0]) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(raw), `"reads_modules":true`) || !strings.Contains(string(raw), `"reads_generation":true`) { - t.Fatalf("the report does not say it reads modules beside the generation: %s", raw) - } -} diff --git a/internal/link/replay331_test.go b/internal/link/replay331_test.go index 468af9f..a603fd3 100644 --- a/internal/link/replay331_test.go +++ b/internal/link/replay331_test.go @@ -13,7 +13,7 @@ import ( // The NATS link is attached with no connection under it, so no bus is needed: the question must reach the // link and fail there, not stop at the queue as if no link were open. func TestReplay331(t *testing.T) { - q := &Queue{Membership: Membership{Node: "laptop"}} + q := &Session{Membership: Membership{Node: "laptop"}} q.attach(t.Context(), &natsLink{node: "laptop"}) _, err := q.AskTool(t.Context(), "sensors", "sensors_health") if err == nil { diff --git a/internal/link/run.go b/internal/link/run.go index 4628640..b19a2a5 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -6,18 +6,9 @@ import ( "encoding/json" "errors" "fmt" - "sort" - "strings" "time" ) -// ErrForged is what a node returns for a declaration whose signature is not the mesh's. -// -// Its own error, and it must never be confused with a malformed message. novox/hq ADR 0004 -// requires a host to tell *this is not from the mesh I joined* apart from *this is malformed*: -// the first means somebody is trying, the second means something is broken. -var ErrForged = errors.New("this declaration was not signed by the mesh this node joined") - // Membership is what a node needs to reach its mesh again, held by the caller. type Membership struct { Node string @@ -30,13 +21,6 @@ type Membership struct { Transport string } -// Applier is what the host does with a declaration that has been proved to come from the mesh. -// -// It receives the signature as well as the declaration, so the host can keep both: what it was -// told is kept signed and verified again when it is read back, which means the file on disk is -// trusted for the same reason the message was rather than for being local. -type Applier func(ctx context.Context, declaration, signature []byte) Report - // Announce is how the link says what is happening, so a node running unattended leaves an // account of it. Nil is allowed and means say nothing. type Announce func(string) @@ -64,35 +48,14 @@ type Announce func(string) // Nil is allowed and means nothing ever rouses it, which is every machine that does not suspend. type Roused <-chan struct{} -func Hold(ctx context.Context, m Membership, queue *Queue, say Announce, timeout time.Duration) error { +func Hold(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration) error { return HoldRoused(ctx, m, queue, say, timeout, nil) } -// Unsaid keeps the report of the last apply until the mesh has taken it, so a report lost between -// applying and publishing — the host standing aside for a successor, a crash, a power cut — is said -// again the next time this node is linked (novox/hq issue 264). Without it the declaration has been -// settled or applied, the machine is what the mesh said, and the mesh waits for a report that the -// node believes is gone. -// -// **Exactly the report the apply made, never a new one.** Re-applying the kept declaration would -// describe the machine as it is now; what the mesh waits for is what was done with what it sent. -// Nil is allowed and keeps nothing. -type Unsaid interface { - // Keep holds the report of an apply until it is said. A report naming no declaration — refused - // before it was read — replaces nothing worth saying, and clears what was kept. - Keep(Report) error - // Said forgets the kept report once the broker has taken the report about the same declaration. - Said(declared string) error - // Pending is the report kept and not yet said, if any. - Pending() (Report, bool, error) -} - // HoldRoused is Hold, told when the machine has reason to think its link is stale. // -// **What arrives is enqueued, never applied here** (novox/hq to-be 45 §6). The queue's one worker -// applies, outlives every link, and says its reports on whichever link is open; the caller runs it -// (Queue.Run) for as long as this holds. -func HoldRoused(ctx context.Context, m Membership, queue *Queue, say Announce, +// The session outlives every link and says the node's facts on whichever is open (novox/hq ADR 0539 §9). +func HoldRoused(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration, roused Roused) error { return holdWith(ctx, func(ctx context.Context) error { @@ -185,11 +148,11 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err } } -// Run holds the link open once, applying what arrives and reporting what happened. +// Run holds the link open once, hearing what the controller asks and saying the node's facts. // // Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason; // Hold is what decides whether to open it again. -func Run(ctx context.Context, m Membership, queue *Queue, say Announce, timeout time.Duration) error { +func Run(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration) error { link, err := Open(ctx, m, timeout) if err != nil { return err @@ -200,7 +163,7 @@ func Run(ctx context.Context, m Membership, queue *Queue, say Announce, timeout // serve is Run on a link already open: separated so what the node says, and when, can be tested // without a broker. -func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Announce, +func serve(ctx context.Context, link Link, m Membership, queue *Session, say Announce, timeout time.Duration) error { if say == nil { say = func(string) {} @@ -209,18 +172,13 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou // Said, because it is the event anybody watching actually wants. Without it a node logs every // failure and nothing on success, so a log full of "trying again" and then silence reads as // still broken when it means the opposite. - say("in the mesh, hearing what this node should be") + say("in the mesh, hearing what the controller asks of this node") - // **What the last apply did, if the mesh never heard it**, is said before anything newly - // delivered can be: the queue says it as the link is handed to it (novox/hq issue 264). And the - // link is not let go while the queue's worker is mid-act: an apply that ends the link — the host - // standing aside for a successor it delivered (novox/hq ADR 0141) — is still reported on it, - // before the deferred close. + // The session says this node's facts as the link is handed to it, and is not let go while it is mid-act. queue.attach(ctx, link) defer queue.detach(link) - declarations := link.Declarations() - // And the mesh asking what this node last applied (to-be 45 §6), on a link that hears the question. + // And the mesh asking this node to say its facts again, on a link that hears the question. var asks <-chan struct{} if a, ok := link.(Asked); ok { asks = a.AskedToReport() @@ -248,7 +206,6 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou } for { - // Asked to stop — by standing aside, say — nothing further is taken in. if ctx.Err() != nil { return nil } @@ -258,90 +215,11 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou case reason := <-link.Lost(): return reason case <-asks: - queue.ReportAsked() - case declaration, ok := <-declarations: - if !ok { - // The link's own reason, when it has managed to say one: "stopped delivering" on - // its own says nothing about why, and why is the whole of what an operator wants. - select { - case reason := <-link.Lost(): - return reason - default: - return errors.New("the mesh stopped sending this node declarations") - } - } - // Whatever else is already waiting goes with it. **Enqueued, not applied**: the queue - // applies the newest of everything delivered and not yet taken, and reports the rest as - // set aside (novox/hq to-be 45 §6). - queue.Deliver(gather(declarations, declaration, drainWindow)...) + queue.FactsAsked(ctx) } } } -// short is a declaration's digest as a person reads it in a log line. -func short(digest string) string { - if len(digest) > 12 { - return digest[:12] - } - return digest -} - -// drainDepth is how many declarations the link holds unread; drainWindow is how long it waits for -// another to follow the one it has. Both small: a push is rare and a backlog is the exception this -// exists for, not the shape of ordinary traffic. -const ( - drainDepth = 16 - drainWindow = 750 * time.Millisecond -) - -// gather takes what is already waiting behind `first`, in the order it arrived. It waits `window` -// for a straggler after each arrival and no longer: a declaration in flight from the mesh arrives -// within that; one that does not is the next push. Which of them is applied is the queue's to decide -// (pick), across everything delivered and not yet taken. -// -// **Its job narrows once declarations are state rather than messages, and does not disappear.** -// On the bus a declaration is last-per-subject (novox/hq design 29 §4), so a node that was away -// receives exactly the current one instead of a queue of superseded ones — the catch-up half is the -// stream's. And a declaration's order decides which is newest, where this window only infers it from -// arrival time, which is the wire-level answer to novox/hq issue 107. -// -// What remains is the live case: three pushes in quick succession to a *connected*, idle node are -// three deliveries, whatever the stream later retains, and gathering them is what makes them one -// apply rather than two. So this is narrowed, not deleted — and saying which half goes is worth more -// than a note that it "can probably be removed", which is how a load-bearing window gets deleted by -// somebody in a hurry. -func gather(arriving <-chan Declaration, first Declaration, window time.Duration) []Declaration { - batch := []Declaration{first} - for { - select { - case next, ok := <-arriving: - if !ok { - return batch - } - batch = append(batch, next) - case <-time.After(window): - return batch - } - } -} - -// handleBody is the whole of deciding whether to trust a message, separated from the broker so it -// can be tested as the security check it is rather than as message plumbing. -func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report { - var signed Signed - if err := json.Unmarshal(body, &signed); err != nil { - return Report{Node: m.Node, Refused: "this message is not a declaration: " + err.Error()} - } - - // Before anything is read out of it, let alone applied. The host applies whatever the link - // delivers, so this check is the difference between the mesh changing this machine and - // anybody changing it. - if !ed25519.Verify(m.Signer, signed.Declaration, signed.Signature) { - return Report{Node: m.Node, Refused: ErrForged.Error()} - } - return apply(ctx, signed.Declaration, signed.Signature) -} - // publishReport tells the mesh what this node did, and says whether the broker took it. // Publish sends one report on this node's own connection and returns: the one-shot path for a // report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same @@ -379,36 +257,3 @@ func publishReport(ctx context.Context, bus Bus, m Membership, report Report, } return true } - -// heldNote is what this apply did NOT do, for the line that says what it did. -// -// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125). -// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but -// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources, -// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took -// opening state.json by hand; not reading it took every public name on the machine down, because the -// operator had four green surfaces and a discrepancy nobody could interpret. -// -// So the line that reports the apply carries it. Grouped by module and ordered by name, because the -// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing -// they can act on — `take` is the verb, and it takes a module. -func heldNote(held []Held) string { - if len(held) == 0 { - return "" - } - byModule := map[string]int{} - for _, h := range held { - byModule[h.Module]++ - } - names := make([]string, 0, len(byModule)) - for name := range byModule { - names = append(names, name) - } - sort.Strings(names) - parts := make([]string, 0, len(names)) - for _, name := range names { - parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name])) - } - return fmt.Sprintf(", %d held until their module is taken (%s)", - len(held), strings.Join(parts, ", ")) -} diff --git a/internal/link/session.go b/internal/link/session.go new file mode 100644 index 0000000..0eb2d80 --- /dev/null +++ b/internal/link/session.go @@ -0,0 +1,257 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program +// that wrote them, beside its epoch and sequence. +const Writer = "node-engine" + +// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks +// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts. +// +// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an +// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles +// a whole declaration. +type Session struct { + // Membership is the node and the mesh's key a controller's ask is verified with. + Membership Membership + // OnLink is run, off the link's own loop, each time a link opens and when the controller asks the node to say its + // facts again. It says them with SayFacts. Nil says nothing. + OnLink func(ctx context.Context) + // Heard is told every facts event the mesh has taken. Nil is allowed. + Heard func(Report) + // HaveWritten is told each time a have key is written, or found already saying what this node-engine would write: + // the first proves a node-engine on trial (novox/hq ADR 0539 §9). Nil is allowed. + HaveWritten func() + // HandOver judges and records a directory used as found handed to the mesh by the operator, asked by + // the controller over the link (novox/hq issue 356). Nil answers every such ask with a refusal. + HandOver HandOverFunc + // SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the + // controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal. + SetuidSearch SetuidSearchFunc + // JudgeDirectories judges this node's declared directories read-only, asked by the controller over the + // link (novox/hq issue 358). Nil answers every such ask with a refusal. + JudgeDirectories JudgeDirectoriesFunc + // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved + // warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal. + AccessAct AccessActFunc + // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of + // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. + MoveSecret MoveSecretFunc + Say Announce + Timeout time.Duration + + once sync.Once + + mu sync.Mutex + bus Bus // the link open now; nil while there is none + + // saying is held while a facts event is made and said, so they leave in the order they are made. + saying sync.Mutex + // epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one + // without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2). + epoch int64 + sequence int64 + + // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). + haves haveState +} + +func (q *Session) init() { + q.once.Do(func() { + if q.Say == nil { + q.Say = func(string) {} + } + q.epoch = time.Now().UnixMilli() + }) +} + +// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them. +// A facts event made with no link open is not kept: the next link says the facts as they are then. +func (q *Session) SayFacts(ctx context.Context, r Report) bool { + q.init() + q.saying.Lock() + defer q.saying.Unlock() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return false + } + q.sequence++ + r.Node = q.Membership.Node + r.Writer = Writer + r.Order = Order{Epoch: q.epoch, Sequence: q.sequence} + if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) { + return false + } + // After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132). + if q.noteApply(r) { + q.writeHaves(ctx, bus) + } + if q.Heard != nil && r.Refused == "" { + q.Heard(r) + } + return true +} + +// FactsAsked is the mesh asking this node to say its facts again: run on the link open now. +func (q *Session) FactsAsked(ctx context.Context) { + q.init() + if q.OnLink != nil { + go q.OnLink(ctx) + } +} + +// attach is a link opening: it is the link facts are said on from now. +func (q *Session) attach(ctx context.Context, bus Bus) { + q.init() + q.mu.Lock() + q.bus = bus + q.mu.Unlock() + q.FactsAsked(ctx) +} + +// WithLink is ctx carrying the link now open, so an apply can fetch a composition's files given by digest (novox/hq ADR +// 0486, issue 1695). A session with no link open carries none. +func (q *Session) WithLink(ctx context.Context) context.Context { + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return ctx + } + return WithFiles(ctx, bus) +} + +// detach is the link closing: nothing more is said on it. +func (q *Session) detach(bus Bus) { + q.init() + q.saying.Lock() + defer q.saying.Unlock() + q.mu.Lock() + if q.bus == bus { + q.bus = nil + } + q.mu.Unlock() +} + +// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the +// machine's networking its engine judges). +func (q *Session) Linked() bool { + q.init() + q.mu.Lock() + defer q.mu.Unlock() + return q.bus != nil +} + +// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether +// the bus took it. A statement is a word about the machine as it is, and it must not wait behind an apply: a +// container crash-looping while a long apply runs is said while it runs. False with no link open, or one that cannot +// say it; the caller says it again on its next look, and the next facts event carries it whatever happens. +func (q *Session) SayHealth(ctx context.Context, h Health) bool { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + sayer, ok := bus.(HealthBus) + if bus == nil || !ok { + return false + } + body, err := json.Marshal(HealthSaid{Node: q.Membership.Node, Health: h}) + if err != nil { + return false + } + saying, cancel := context.WithTimeout(ctx, q.timeout()) + defer cancel() + if err := sayer.Health(saying, q.Membership.Node, body); err != nil { + q.Say("could not say this machine's health: " + err.Error()) + return false + } + // A look that changed a module's health changes its have (issue 1132). + if q.noteHealth(h) { + q.writeHaves(ctx, bus) + } + return true +} + +func (q *Session) timeout() time.Duration { + if q.Timeout <= 0 { + return 10 * time.Second + } + return q.Timeout +} + +// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR +// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open +// or nothing answered — said as the check's failure, never as healthy. +func (q *Session) AskTool(ctx context.Context, module, tool string) (health.Answer, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return health.Answer{}, errors.New("no link to the bus is open") + } + asker, ok := bus.(ToolBus) + if !ok { + return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool") + } + reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}")) + if err != nil { + return health.Answer{}, err + } + return ReadToolAnswer(reply) +} + +// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over +// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for. +func (q *Session) AskController(ctx context.Context, body []byte) ([]byte, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + asker, ok := bus.(ToolBus) + if !ok { + return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked) + } + return asker.Ask(ctx, CLISubject(q.Membership.Node), body) +} + +// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any +// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272). +var ErrNotAsked = errors.New("not asked") + +// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call — +// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the +// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the +// SDK cannot check is unhealthy too, so a malformed one never excuses anything. +func ReadToolAnswer(reply []byte) (health.Answer, error) { + var env struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + if err := json.Unmarshal(reply, &env); err != nil { + return health.Answer{}, fmt.Errorf("its answer is not one: %w", err) + } + if env.Error != "" { + return health.Unhealthy(env.Error), nil + } + a, err := health.Read(env.Result) + if err != nil { + return health.Unhealthy("it answered something other than healthy or not"), nil + } + return a, nil +} diff --git a/internal/link/session_test.go b/internal/link/session_test.go new file mode 100644 index 0000000..cd421a2 --- /dev/null +++ b/internal/link/session_test.go @@ -0,0 +1,164 @@ +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled, +// as the bus's is ("reporting: context canceled"), or while refusing. +type quietLink struct { + mu sync.Mutex + reports []Report + refusing bool + lost chan error +} + +func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} } + +func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error { + if err := ctx.Err(); err != nil { + return err + } + l.mu.Lock() + defer l.mu.Unlock() + if l.refusing { + return errors.New("refused") + } + var r Report + if err := json.Unmarshal(body, &r); err != nil { + return err + } + l.reports = append(l.reports, r) + return nil +} +func (l *quietLink) Lost() <-chan error { return l.lost } +func (l *quietLink) Close() {} + +func (l *quietLink) said() []Report { + l.mu.Lock() + defer l.mu.Unlock() + return append([]Report(nil), l.reports...) +} + +func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { + t.Helper() + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return Membership{Node: "n1", Signer: public}, private +} + +// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539 +// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts. +func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) { + m, _ := aMember(t) + l := newQuietLink() + s := &Session{Membership: m, Timeout: time.Second} + s.attach(context.Background(), l) + + if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) { + t.Fatal("the facts were not taken") + } + got := l.said() + if len(got) != 2 { + t.Fatalf("%d facts events were said", len(got)) + } + for _, r := range got { + if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 { + t.Fatalf("the facts do not say who wrote them: %+v", r) + } + } + if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 { + t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order) + } + + // A later start orders after this one with nothing kept on disk. + time.Sleep(2 * time.Millisecond) + later := &Session{Membership: m, Timeout: time.Second} + later.attach(context.Background(), l) + later.SayFacts(context.Background(), Report{}) + if last := l.said()[2]; last.Epoch <= got[1].Epoch { + t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch) + } +} + +// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then. +func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { + m, _ := aMember(t) + l := newQuietLink() + s := &Session{Membership: m, Timeout: time.Second} + if s.SayFacts(context.Background(), Report{}) { + t.Fatal("facts were said with no link") + } + s.attach(context.Background(), l) + s.detach(l) + if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 { + t.Fatal("facts were said on a link that was let go") + } +} + +// A link opening says the node's facts, off the link's own loop. +func TestALinkOpeningSaysTheFacts(t *testing.T) { + m, _ := aMember(t) + l := newQuietLink() + opened := make(chan struct{}, 1) + s := &Session{Membership: m, Timeout: time.Second, OnLink: func(context.Context) { opened <- struct{}{} }} + s.attach(context.Background(), l) + select { + case <-opened: + case <-time.After(2 * time.Second): + t.Fatal("a link opening did not ask for the facts") + } +} + +// aBus is a connection to a real server, because some claims here are about one: set MESH_TEST_NATS to a server with +// JetStream, or the test is skipped. +func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, + Retention: nats.WorkQueuePolicy} + if _, err := js.StreamInfo(want.Name); err != nil { + if _, err := js.AddStream(want); err != nil { + t.Fatal(err) + } + } + if err := js.PurgeStream(want.Name); err != nil { + t.Fatal(err) + } + return conn, js +} + +// eventuallyHeld waits for a condition the bus reaches on its own time. +func eventuallyHeld(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatal(what) + } + time.Sleep(5 * time.Millisecond) + } +} diff --git a/internal/link/unsaid_test.go b/internal/link/unsaid_test.go deleted file mode 100644 index 13257fd..0000000 --- a/internal/link/unsaid_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package link - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "errors" - "reflect" - "sync" - "testing" - "time" -) - -// A report of an apply reaches the mesh even when the apply ended the link, and one lost anyway is -// said again the next time the node is linked (novox/hq issue 264). - -// quietLink is a link with no broker behind it. A report is refused once the context it is -// published on is cancelled, as the bus's is ("reporting: context canceled"), or while refusing. -type quietLink struct { - mu sync.Mutex - reports []Report - refusing bool - delivered chan Declaration - lost chan error -} - -func newQuietLink(arriving ...Declaration) *quietLink { - l := &quietLink{delivered: make(chan Declaration, len(arriving)), lost: make(chan error, 1)} - for _, d := range arriving { - l.delivered <- d - } - return l -} - -func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error { - if err := ctx.Err(); err != nil { - return err - } - l.mu.Lock() - defer l.mu.Unlock() - if l.refusing { - return errors.New("refused") - } - var r Report - if err := json.Unmarshal(body, &r); err != nil { - return err - } - l.reports = append(l.reports, r) - return nil -} -func (l *quietLink) Declarations() <-chan Declaration { return l.delivered } -func (l *quietLink) Lost() <-chan error { return l.lost } -func (l *quietLink) Close() {} - -func (l *quietLink) said() []Report { - l.mu.Lock() - defer l.mu.Unlock() - return append([]Report(nil), l.reports...) -} - -// keptInMemory is Unsaid without a disk. -type keptInMemory struct { - mu sync.Mutex - report *Report -} - -func (k *keptInMemory) Keep(r Report) error { - k.mu.Lock() - defer k.mu.Unlock() - if r.Declared == "" { - k.report = nil - return nil - } - k.report = &r - return nil -} -func (k *keptInMemory) Said(declared string) error { - k.mu.Lock() - defer k.mu.Unlock() - if k.report != nil && k.report.Declared == declared { - k.report = nil - } - return nil -} -func (k *keptInMemory) Pending() (Report, bool, error) { - k.mu.Lock() - defer k.mu.Unlock() - if k.report == nil { - return Report{}, false, nil - } - return *k.report, true, nil -} - -func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { - t.Helper() - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - return Membership{Node: "n1", Signer: public}, private -} - -// Measured on the anchor: a declaration carried a new controller and a new host, the host applied -// it, stood aside for its successor — which cancels the link — and the report of that apply failed -// with "context canceled". The plan waited on it until somebody pushed by hand. -// -// R3 of novox/hq to-be 45 §9, as a unit: the node-engine self-updates during its report, and the -// report still arrives — on the link the apply came over, before it is let go. -func TestAnApplyThatEndsTheLinkIsStillReported(t *testing.T) { - m, key := aMember(t) - declaration := &said{body: signedBy(t, key, []byte(`{"declaration":1}`))} - behind := &said{body: signedBy(t, key, []byte(`{"declaration":2}`))} - l := newQuietLink(declaration) - kept := &keptInMemory{} - - ctx, standAside := context.WithCancel(context.Background()) - defer standAside() - applied := 0 - q := aQueue(m, func(context.Context, []byte, []byte) Report { - applied++ - standAside() // the host delivered its successor, and stands aside for it - return Report{Declared: "d1", Applied: []string{"a"}} - }, kept) - worker := runWorker(ctx, q) - - done := make(chan error, 1) - go func() { done <- serve(ctx, l, m, q, nil, time.Second) }() - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(5 * time.Second): - t.Fatal("the link did not end after the apply stood aside") - } - // Something delivered after the host stood aside is not applied by it: the successor will be - // sent it again. - q.Deliver(behind) - <-worker - - reports := l.said() - if len(reports) != 1 || reports[0].Declared != "d1" || !reflect.DeepEqual(reports[0].Applied, []string{"a"}) { - t.Fatalf("the apply's report did not reach the mesh: %+v", reports) - } - if !declaration.wasHandled() { - t.Fatal("the declaration was not settled after its report") - } - if applied != 1 || behind.wasHandled() { - t.Fatalf("%d applies, and the declaration behind settled %v: after standing aside nothing "+ - "further is applied", applied, behind.wasHandled()) - } - if _, ok, _ := kept.Pending(); ok { - t.Fatal("a report the mesh took is still kept to be said again") - } -} - -// A report that did not reach the mesh — the host died between applying and publishing, or the -// broker refused it — is said on the next link, before anything else, and exactly as it was made. -func TestAReportLostAfterTheApplyIsSaidOnTheNextLink(t *testing.T) { - m, key := aMember(t) - made := Report{Declared: "d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"}} - kept := &keptInMemory{} - - // The apply happens and its report is lost. - first := newQuietLink(&said{body: signedBy(t, key, []byte(`{"declaration":1}`))}) - first.refusing = true - ctx, stop := context.WithCancel(context.Background()) - q := aQueue(m, func(context.Context, []byte, []byte) Report { stop(); return made }, kept) - worker := runWorker(ctx, q) - if err := serve(ctx, first, m, q, nil, time.Second); err != nil { - t.Fatal(err) - } - <-worker - if len(first.said()) != 0 { - t.Fatal("the broker refused the report and it counted as said") - } - lost, ok, _ := kept.Pending() - if !ok || lost.ReportSequence != 1 { - t.Fatalf("the lost report was not kept as it was made: %+v", lost) - } - - // The next host links. It is stopped at once, so all it does is what it does on linking. - next := newQuietLink() - gone, cancel := context.WithCancel(context.Background()) - cancel() - never := func(context.Context, []byte, []byte) Report { - t.Fatal("nothing was delivered, and something was applied") - return Report{} - } - if err := serve(gone, next, m, aQueue(m, never, kept), nil, time.Second); err != nil { - t.Fatal(err) - } - reports := next.said() - if len(reports) != 1 || !reflect.DeepEqual(reports[0], lost) { - t.Fatalf("the lost report was not said again as it was made: %+v", reports) - } - if _, ok, _ := kept.Pending(); ok { - t.Fatal("a report said again is still kept") - } -} - -// A node that never applied anything, or whose last report was taken, says nothing on linking. -func TestNothingIsSaidAgainWhenNothingWasLost(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - gone, cancel := context.WithCancel(context.Background()) - cancel() - if err := serve(gone, l, m, aQueue(m, nil, &keptInMemory{}), nil, time.Second); err != nil { - t.Fatal(err) - } - if err := serve(gone, l, m, aQueue(m, nil, nil), nil, time.Second); err != nil { - t.Fatal(err) - } - if reports := l.said(); len(reports) != 0 { - t.Fatalf("a node with nothing lost reported %+v", reports) - } -} diff --git a/internal/link/witnessing.go b/internal/link/witnessing.go index 340eaec..4b1f8ad 100644 --- a/internal/link/witnessing.go +++ b/internal/link/witnessing.go @@ -48,7 +48,7 @@ func DirectGetSubject(bucket, key string) string { func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id } // Asker is the link open now, when there is one and it can ask. -func (q *Queue) Asker() Asker { +func (q *Session) Asker() Asker { q.init() q.mu.Lock() defer q.mu.Unlock() diff --git a/internal/store/numbers.go b/internal/store/numbers.go deleted file mode 100644 index 61fe263..0000000 --- a/internal/store/numbers.go +++ /dev/null @@ -1,94 +0,0 @@ -package store - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" -) - -// What the node-engine counts about what it says, kept so the counting outlives the process. -// -// **The report sequence must increase across restarts and self-updates** (novox/hq to-be 45 §6). The -// mesh keeps, per machine, the highest report it accepted and refuses an older one; a host that began -// again from one after every restart — and a self-update is a restart — would have every report it -// made refused until it had counted past where its predecessor stopped. So the number is kept beside -// the state, where the successor reads it. -// -// And the count of declarations refused as older than what this node applied (rule 2), which the -// mesh's stale-writer watchdog reads from every report. - -// NumbersName is where they live, beside the state. -const NumbersName = "numbers.json" - -// NumbersPath is where the numbers live, given where the state lives. -func NumbersPath(statePath string) string { - return filepath.Join(filepath.Dir(statePath), NumbersName) -} - -// Numbers is what is kept. -type Numbers struct { - // ReportSequence is the number of the last report this node made. - ReportSequence int64 `json:"report_sequence"` - // RefusedOlder is how many declarations it has refused as older, ever. - RefusedOlder int64 `json:"refused_older"` -} - -// ReadNumbers is what was kept, or zero when nothing was — a node that has never reported. -// -// **Unreadable is an error, never zero** (novox/hq ADR 0227, rule 4): read as zero, every report the -// node makes next would be older than the ones it already made, and refused. -func ReadNumbers(path string) (Numbers, error) { - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return Numbers{}, nil - } - if err != nil { - return Numbers{}, err - } - var n Numbers - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.DisallowUnknownFields() - if err := dec.Decode(&n); err != nil { - return Numbers{}, fmt.Errorf("the numbers kept at %s are unreadable: %w", path, err) - } - if n.ReportSequence < 0 || n.RefusedOlder < 0 { - return Numbers{}, fmt.Errorf("the numbers kept at %s are below zero, which nothing counting writes", path) - } - return n, nil -} - -// SaveNumbers keeps them, replacing what was kept, and is on disk when it returns: a number used and -// not kept is one the next host would use again. -func SaveNumbers(path string, n Numbers) error { - raw, err := json.Marshal(n) - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err - } - tmp, err := os.CreateTemp(filepath.Dir(path), ".numbers-*") - if err != nil { - return err - } - defer os.Remove(tmp.Name()) - if err := tmp.Chmod(0o600); err != nil { - tmp.Close() - return err - } - if _, err := tmp.Write(raw); err != nil { - tmp.Close() - return err - } - if err := tmp.Sync(); err != nil { - tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(tmp.Name(), path) -} diff --git a/internal/store/unsaid.go b/internal/store/unsaid.go deleted file mode 100644 index 4412b19..0000000 --- a/internal/store/unsaid.go +++ /dev/null @@ -1,76 +0,0 @@ -package store - -import ( - "errors" - "os" - "path/filepath" -) - -// The report of the last apply of a declaration from the mesh, kept until the mesh has taken it. -// -// **An apply the mesh never heard about is one it waits on for ever.** The controller's release -// plan waits for a machine to report the exact declaration it was sent; a report lost between the -// apply and the publish — a host standing aside for its successor, a crash, a power cut — leaves the -// machine applied and the plan waiting until somebody pushes by hand (novox/hq issue 264). The -// declaration itself is kept (declared.go), and re-applying it would say something new about the -// machine; what was lost is what *that* apply did, so that is what is kept, exactly as it was made, -// and said again once the host is linked. -// -// Opaque bytes here: the report is the link's word, and the store keeps it without reading it. - -// UnsaidName is where it lives, beside the state. -const UnsaidName = "unsaid.json" - -// UnsaidPath is where the unsaid report lives, given where the state lives. -func UnsaidPath(statePath string) string { - return filepath.Join(filepath.Dir(statePath), UnsaidName) -} - -// SaveUnsaid keeps a report the mesh has not taken yet, replacing whatever was kept before: only -// the last apply's report is worth saying again, because the mesh compares against what it sent last. -func SaveUnsaid(path string, report []byte) error { - if len(report) == 0 { - return errors.New("refusing to keep an empty report") - } - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err - } - tmp, err := os.CreateTemp(filepath.Dir(path), ".unsaid-*") - if err != nil { - return err - } - defer os.Remove(tmp.Name()) - if err := tmp.Chmod(0o600); err != nil { - tmp.Close() - return err - } - if _, err := tmp.Write(report); err != nil { - tmp.Close() - return err - } - if err := tmp.Sync(); err != nil { - tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(tmp.Name(), path) -} - -// ReadUnsaid is the report kept unsaid, or nil when there is none — the ordinary case. -func ReadUnsaid(path string) ([]byte, error) { - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } - return raw, err -} - -// ClearUnsaid forgets the kept report, once the mesh has taken it. Absent is already clear. -func ClearUnsaid(path string) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return nil -}