diff --git a/cmd/mesh-host/applywhole_test.go b/cmd/mesh-host/applywhole_test.go new file mode 100644 index 0000000..f332756 --- /dev/null +++ b/cmd/mesh-host/applywhole_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// applyAndKeep applies a whole declaration the way the carried bundle is applied, and says what came of it as the facts +// the node says: what the tests of the apply's steps read. The node-engine applies no whole declaration on its own now +// (novox/hq ADR 0539 §9); this is the test's way to put a machine in a state. +func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared, sched *apply.Scheduler, + say link.Announce) link.Report { + declared, err := declaration.Parse(raw) + if err != nil { + return link.Report{Refused: err.Error()} + } + got := applyModuleHeld(ctx, opts, declared, nil, sched, say) + if got.Refused != "" { + return link.Report{Refused: got.Refused} + } + held := map[string]bool{} + for _, h := range got.Updated.Held { + held[h.ID] = true + } + report := factsOf(ctx, opts, declared, got.Updated, got.Outcome, held) + if signed != nil { + if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil { + report.Failed = map[string]string{"keeping the declaration": err.Error()} + } + } + if got.Err != nil { + report.Failed = map[string]string{"apply": got.Err.Error()} + } + report.Failed = withUnanswered(report.Failed, got.Probe) + report.Unapplied = unapplied(declared, got.Outcome, got.Err) + return report +} diff --git a/cmd/mesh-host/backup_event_test.go b/cmd/mesh-host/backup_event_test.go new file mode 100644 index 0000000..0a773cc --- /dev/null +++ b/cmd/mesh-host/backup_event_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +func backupEvent(node, module string, ok bool, at time.Time, why string) []byte { + raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "ok": ok, "at": at, "error": why}) + return raw +} + +// A backup is done when the holder says so for this node and module, held on its words when it says it failed, and held +// when it says nothing within the bound; another node's or module's event, or one from before the ask, is not the answer +// (R6-A A5, novox/hq ADR 0429 §14). +func TestABackupEndsWhenItsEventSaysSo(t *testing.T) { + since := time.Now() + run := func(within time.Duration, events ...[]byte) (done bool, words string) { + ch := make(chan []byte, len(events)) + for _, e := range events { + ch <- e + } + got := awaitBackup(context.Background(), ch, "n1", "db", since, within) + return got.Done, got.Words + } + if done, _ := run(time.Second, + backupEvent("n2", "db", true, since, ""), backupEvent("n1", "web", true, since, ""), + backupEvent("n1", "db", false, since.Add(-time.Hour), "an old night"), + backupEvent("n1", "db", true, since.Add(time.Second), "")); !done { + t.Error("the event of this node and module was not the answer") + } + if done, words := run(time.Second, backupEvent("n1", "db", false, since, "the repository is locked")); done || + !strings.Contains(words, "repository is locked") { + t.Errorf("a failed backup: done=%v words=%q", done, words) + } + if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") { + t.Errorf("silence within the bound: done=%v words=%q", done, words) + } +} diff --git a/cmd/mesh-host/busy_test.go b/cmd/mesh-host/busy_test.go index b1edaea..a7cac9c 100644 --- a/cmd/mesh-host/busy_test.go +++ b/cmd/mesh-host/busy_test.go @@ -5,62 +5,30 @@ import ( "sync/atomic" "testing" "time" - - "github.com/novox/mesh-host/internal/link" ) -// A container waiting because it said busy is news when it starts waiting and when it is replaced, never on -// its output alone (novox/hq ADR 0335): the walk waiting on this machine starts its gate at the report that -// says it replaced, and a reconcile is otherwise silent. -func TestABusyWaitIsSaidWhenItStartsAndWhenItEnds(t *testing.T) { - quiet := link.Report{Outward: []string{"eth0"}} - waiting := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server", - Container: "mesh-build-agent", State: "waiting", Said: "building a"}}} - still := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server", - Container: "mesh-build-agent", State: "waiting", Said: "building b, a minute on"}}} - replaced := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server", - Container: "mesh-build-agent", State: "replaced-when-idle"}}} - if adoptionFingerprint(waiting) == adoptionFingerprint(quiet) { - t.Error("a container starting to wait is not news") - } - if adoptionFingerprint(still) != adoptionFingerprint(waiting) { - t.Error("a wait whose output changed reads as news on every ask") - } - if adoptionFingerprint(replaced) == adoptionFingerprint(waiting) { - t.Error("a container replaced after it waited is not news") - } - if !worthSaying(link.Report{Busy: replaced.Busy}) { - t.Error("a replacement after a wait is not worth saying") - } -} - -// While a container waits, a pass is asked for every so often, not at the reconcile's five minutes; with -// nothing waiting, none (novox/hq ADR 0335 §2). -func TestAWaitIsAskedAgainSoonerThanTheReconcile(t *testing.T) { - if BusyAskEvery >= ReconcileEvery || BusyAskEvery > time.Minute { +// While a container waits, a pass is asked for every so often, not at some long interval; with nothing waiting, none +// (novox/hq ADR 0335 §2, ADR 0360 §11). +func TestAWaitIsAskedAgainSoon(t *testing.T) { + if BusyAskEvery > time.Minute { t.Fatalf("a waiting container is asked every %s", BusyAskEvery) } - var reconciles atomic.Int32 - queue := &link.Queue{Reconcile: func(context.Context) (link.Report, bool) { - reconciles.Add(1) - return link.Report{}, false - }} + var asked atomic.Int32 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go queue.Run(ctx) busyWaiting.Store(false) t.Cleanup(func() { busyWaiting.Store(false) }) - go busyPasses(ctx, queue, 10*time.Millisecond) + go busyPasses(ctx, func() { asked.Add(1) }, 10*time.Millisecond) time.Sleep(100 * time.Millisecond) - if n := reconciles.Load(); n != 0 { + if n := asked.Load(); n != 0 { t.Fatalf("%d passes were asked with nothing waiting", n) } busyWaiting.Store(true) deadline := time.Now().Add(2 * time.Second) - for reconciles.Load() == 0 && time.Now().Before(deadline) { + for asked.Load() == 0 && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) } - if reconciles.Load() == 0 { + if asked.Load() == 0 { t.Fatal("no pass was asked while a container waits") } } diff --git a/cmd/mesh-host/epoch_test.go b/cmd/mesh-host/epoch_test.go deleted file mode 100644 index 56bb0bc..0000000 --- a/cmd/mesh-host/epoch_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "errors" - "os" - "path/filepath" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be -// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node -// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before -// the machine is read or touched, with a report naming what was refused and what is held, and the -// kept declaration left as it was. -func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) { - dir := t.TempDir() - opts := options{state: filepath.Join(dir, "state.json")} - _, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - target := filepath.Join(dir, "a.conf") - declared := func(epoch, sequence int64) store.Declared { - body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence, - "resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}}) - if err != nil { - t.Fatal(err) - } - return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)} - } - held := declared(57, 3) - if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil { - t.Fatal(err) - } - - stale := declared(41, 12) - report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil) - if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) || - report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) || - report.Refused == "" { - t.Fatalf("the refusal does not name what was refused and what is held: %+v", report) - } - if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { - t.Fatal("the refused declaration touched the machine") - } - kept, err := store.ReadDeclared(store.DeclaredPath(opts.state)) - if err != nil || string(kept.Declaration) != string(held.Declaration) { - t.Fatal("the refused declaration replaced what this node kept") - } -} - -// The queue's counts are kept beside the state and read back by the next host; unreadable is an -// error, never zero. -func TestTheNumbersSurviveTheHost(t *testing.T) { - state := filepath.Join(t.TempDir(), "state.json") - if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 { - t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err) - } - if err := numbersBeside(state).Save(41, 2); err != nil { - t.Fatal(err) - } - if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 { - t.Fatalf("read back %d, %d, %v", sequence, refused, err) - } - if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil { - t.Fatal(err) - } - if _, _, err := numbersBeside(state).Read(); err == nil { - t.Fatal("unreadable numbers were read as zero") - } -} diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index ff84109..28bcc9d 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -27,11 +27,7 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine { return &filling.Engine{ Declared: func() (*declaration.Declaration, error) { - raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) - if err != nil { - return nil, err - } - return declaration.Parse(raw) + return declaredNow(opts.state, signer) }, UIDOf: filling.UIDOf, Lock: func() (func(), error) { diff --git a/cmd/mesh-host/generation_floor_test.go b/cmd/mesh-host/generation_floor_test.go deleted file mode 100644 index 81ec418..0000000 --- a/cmd/mesh-host/generation_floor_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "context" - "testing" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/store" - "github.com/novox/mesh-host/internal/system" -) - -// **The generation applied is kept, and a put-back never lowers it**: after a put-back of generation 5 -// over 7, a send from generation 6 is still older than what this machine applied. -func TestThePutBackDoesNotLowerTheGenerationApplied(t *testing.T) { - built, err := system.For("arch") - if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { - t.Skip("this machine is not one these tests can apply on") - } - was := builtFor - builtFor = "arch" - t.Cleanup(func() { builtFor = was }) - f := machineThatApplied(t, 7) - - newer := f.declared(t, map[string]any{"sequence": 9, "generation": 8}, "docker.conf", f.stays) - if r := applyAndKeep(context.Background(), f.opts, newer.Declaration, &newer, nil, nil); r.Refused != "" { - t.Fatalf("a newer generation was refused: %s", r.Refused) - } - back := f.declared(t, map[string]any{"sequence": 10, "generation": 5, "put_back": true}, "docker.conf", f.stays) - if r := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil); r.Refused != "" { - t.Fatalf("a put-back was refused: %s", r.Refused) - } - state, err := store.Load(f.opts.state) - if err != nil || state.Generation != 8 { - t.Fatalf("the generation applied is %d after a put-back of 5 over 8 (%v)", state.Generation, err) - } - stale := f.declared(t, map[string]any{"sequence": 11, "generation": 7}, "docker.conf", f.stays) - if r := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil); r.OlderGeneration == nil { - t.Fatalf("a send from generation 7 after 8 was applied because a put-back came between: %+v", r) - } -} diff --git a/cmd/mesh-host/generation_test.go b/cmd/mesh-host/generation_test.go deleted file mode 100644 index a9b02f5..0000000 --- a/cmd/mesh-host/generation_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package main - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// The assignment generation a declaration was composed from (novox/hq issue 234). -// -// On 2026-10-04 a declaration newer in sequence than every other, and composed from a view of the -// assignments older than the one this machine had applied, named four fewer modules — and the host, -// ordering by sequence alone, undeclared all four. The sequence orders arrival; it cannot tell a later -// send carrying earlier content. The generation can: the node-engine refuses a declaration composed from -// an older generation than the one it applied, before anything is read from the machine. -// -// These read what the report says through its JSON, as the controller does, so they run against a -// node-engine that has never heard of a generation and fail there for that reason. - -// generationFixture is a machine that applied generation 7: a file the mesh declared, kept in its state, -// and the declaration it kept — signed — that named it. -type generationFixture struct { - opts options - private ed25519.PrivateKey - kept store.Declared - // stays is the file the older generation's declaration does not name. - stays string -} - -func machineThatApplied(t *testing.T, generation int64) generationFixture { - t.Helper() - dir := t.TempDir() - opts := options{state: filepath.Join(dir, "state.json")} - _, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - stays := filepath.Join(dir, "docker.conf") - if err := os.WriteFile(stays, []byte("kept\n"), 0o644); err != nil { - t.Fatal(err) - } - // The state as a node-engine that applied generation 7 keeps it: the file it owns, and the highest - // generation it applied. Written as JSON, so the fixture is the same bytes for any build under test. - state := map[string]any{"resources": []any{map[string]any{"id": "docker.conf", "type": "file", - "origin": store.OriginDeclared, "target": stays}}, "generation": generation} - raw, err := json.Marshal(state) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(opts.state, raw, 0o600); err != nil { - t.Fatal(err) - } - f := generationFixture{opts: opts, private: private, stays: stays} - f.kept = f.declared(t, map[string]any{"sequence": 3, "generation": generation}, "docker.conf", stays) - if err := store.SaveDeclared(store.DeclaredPath(opts.state), f.kept); err != nil { - t.Fatal(err) - } - return f -} - -// declared is a signed declaration with the envelope's keys given and one file. -func (f generationFixture) declared(t *testing.T, envelope map[string]any, id, path string) store.Declared { - t.Helper() - body := map[string]any{"declaration": 1, - "resources": []any{map[string]any{"id": id, "type": "file", "path": path, "content": "x\n"}}} - for k, v := range envelope { - body[k] = v - } - raw, err := json.Marshal(body) - if err != nil { - t.Fatal(err) - } - return store.Declared{Declaration: raw, Signature: ed25519.Sign(f.private, raw)} -} - -// reportJSON is a report as the controller reads it. -func reportJSON(t *testing.T, r link.Report) map[string]any { - t.Helper() - raw, err := json.Marshal(r) - if err != nil { - t.Fatal(err) - } - var out map[string]any - if err := json.Unmarshal(raw, &out); err != nil { - t.Fatal(err) - } - return out -} - -// **A newer sequence from an older generation is refused, and nothing is undeclared** (issue 234). -func TestANewerSequenceFromAnOlderGenerationIsRefusedAndNothingIsUndeclared(t *testing.T) { - f := machineThatApplied(t, 7) - other := filepath.Join(filepath.Dir(f.opts.state), "sudo.conf") - stale := f.declared(t, map[string]any{"sequence": 9, "generation": 6}, "sudo.conf", other) - - report := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil) - said := reportJSON(t, report) - older, _ := said["older_generation"].(map[string]any) - if older == nil || older["generation"] != float64(6) || older["applied"] != float64(7) { - t.Fatalf("the refusal does not name the generation refused and the one applied: %v", said) - } - if report.Refused == "" || !strings.Contains(report.Refused, "generation 6") || - !strings.Contains(report.Refused, "generation 7") { - t.Errorf("the refusal is not said in words naming both generations: %q", report.Refused) - } - if report.Declared != digestOf(stale.Declaration) || report.Sequence != 9 { - t.Errorf("the refusal does not name the declaration refused, so the mesh cannot name who sent it: %+v", report) - } - if _, err := os.Stat(f.stays); err != nil { - t.Fatalf("a refused declaration undeclared what the machine applied: %v", err) - } - if _, err := os.Stat(other); err == nil { - t.Error("the refused declaration touched the machine") - } - state, err := store.Load(f.opts.state) - if err != nil { - t.Fatal(err) - } - if _, owned := state.Find("docker.conf"); !owned { - t.Error("the refused declaration took a resource out of what this machine owns") - } - kept, err := store.ReadDeclared(store.DeclaredPath(f.opts.state)) - if err != nil || string(kept.Declaration) != string(f.kept.Declaration) { - t.Error("the refused declaration replaced what this node kept, so the next reconcile would apply it") - } -} - -// refusedForItsGeneration says a report refused its declaration for the generation it came from. -func refusedForItsGeneration(t *testing.T, r link.Report) bool { - t.Helper() - return reportJSON(t, r)["older_generation"] != nil || strings.Contains(r.Refused, "generation") -} - -// refusedAsUnread says a report refused its declaration as one this node-engine cannot read. -func refusedAsUnread(r link.Report) bool { return strings.Contains(r.Refused, "not a declaration") } - -// **A gate's put-back is applied, whatever generation it carries**: it carries the generation of what it -// puts back, and says it is a put-back (issue 234). -func TestAPutBackFromAnOlderGenerationIsNotRefused(t *testing.T) { - f := machineThatApplied(t, 7) - back := f.declared(t, map[string]any{"sequence": 9, "generation": 5, "put_back": true}, "docker.conf", f.stays) - report := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil) - if refusedForItsGeneration(t, report) || refusedAsUnread(report) { - t.Fatalf("a put-back was refused: %s", report.Refused) - } -} - -// **A declaration that carries no generation is applied as it always was**: an older controller sends -// none, and a node-engine that refused it would strand every machine the moment the controller is older. -func TestADeclarationWithoutAGenerationIsNotRefusedForOne(t *testing.T) { - f := machineThatApplied(t, 7) - older := f.declared(t, map[string]any{"sequence": 9}, "docker.conf", f.stays) - report := applyAndKeep(context.Background(), f.opts, older.Declaration, &older, nil, nil) - if refusedForItsGeneration(t, report) || refusedAsUnread(report) { - t.Fatalf("a declaration without a generation was refused: %s", report.Refused) - } - // And the same generation again is no older. - same := f.declared(t, map[string]any{"sequence": 10, "generation": 7}, "docker.conf", f.stays) - report = applyAndKeep(context.Background(), f.opts, same.Declaration, &same, nil, nil) - if refusedForItsGeneration(t, report) || refusedAsUnread(report) { - t.Fatalf("a declaration of the generation applied was refused: %s", report.Refused) - } -} diff --git a/cmd/mesh-host/judge_directories.go b/cmd/mesh-host/judge_directories.go index c65d19f..2eb2c07 100644 --- a/cmd/mesh-host/judge_directories.go +++ b/cmd/mesh-host/judge_directories.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" @@ -13,16 +12,12 @@ import ( // judgeDirectories judges every directory of the declaration this node keeps, read-only (novox/hq issue 358): which // the walk that follows no link another account could make would refuse. Nothing is made, chmodded or chowned. func judgeDirectories(statePath string) (string, error) { - kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + d, err := declaredNow(statePath, nil) if errors.Is(err, store.ErrNothingDeclared) { return "", errors.New("this node keeps no declaration, so it is declared no directory") } if err != nil { - return "", err - } - d, err := declaration.Parse(kept.Declaration) - if err != nil { - return "", fmt.Errorf("the declaration this node keeps does not read: %w", err) + return "", fmt.Errorf("what runs on this node does not read: %w", err) } return apply.JudgeDirectories(d), nil } diff --git a/cmd/mesh-host/libraries.go b/cmd/mesh-host/libraries.go index 040eb2c..f2baefb 100644 --- a/cmd/mesh-host/libraries.go +++ b/cmd/mesh-host/libraries.go @@ -251,16 +251,12 @@ func startKeeping(ctx context.Context, say func(string)) { // declaration this node keeps, and run beside the applies. func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc { return func(act link.AccessAct) (string, error) { - kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + d, err := declaredNow(statePath, nil) if errors.Is(err, store.ErrNothingDeclared) { return "", errors.New("this node keeps no declaration, so it is declared no library") } if err != nil { - return "", err - } - d, err := declaration.Parse(kept.Declaration) - if err != nil { - return "", fmt.Errorf("the declaration this node keeps does not read: %w", err) + return "", fmt.Errorf("what runs on this node does not read: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/cmd/mesh-host/local_network_test.go b/cmd/mesh-host/local_network_test.go deleted file mode 100644 index 3a7af99..0000000 --- a/cmd/mesh-host/local_network_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/outward" - "github.com/novox/mesh-host/internal/profile" -) - -func reportOn(networks ...outward.Network) link.Report { - return link.Report{Outward: []string{"eth0"}, - Profile: profileAsReported(profile.Profile{Machine: &profile.Machine{Networks: networks}})} -} - -// **A change of the networks a machine is attached to is said unasked** (novox/hq ADR 0340 §7): a port -// reached from the local network is admitted from them, so a laptop that joins another network is said at -// the next reconcile, not at the next push — and the same networks again are not news. -func TestAChangedNetworkIsSaidUnasked(t *testing.T) { - w := &adoptionWatch{} - home := reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"}) - w.said(home) - if w.differs(reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})) { - t.Fatal("the same networks read as a change, so the machine would speak on every reconcile") - } - if !w.differs(reportOn(outward.Network{Link: "eth0", Network: "10.20.0.0/16"})) { - t.Fatal("a machine that moved to another network does not say so until it is pushed to") - } -} diff --git a/cmd/mesh-host/machine_refresh_test.go b/cmd/mesh-host/machine_refresh_test.go deleted file mode 100644 index 91c2dda..0000000 --- a/cmd/mesh-host/machine_refresh_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-host/internal/link" -) - -// Defends novox/hq issue 160: a machine nobody pushed to still says what it is made of once a day, so the -// newest of it the mesh holds is no older than a day. Before, the reconcile said it only with something -// else (a hold, a firewall, an outward link changed), and a converged machine's picture was as old as the -// last delivery to it. -func TestAMachineNobodyPushedToSaysWhatItIsOnceADay(t *testing.T) { - at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) - w := &adoptionWatch{now: func() time.Time { return at }} - quiet := link.Report{Outward: []string{"wlan0"}, - Profile: map[string]any{"machine": map[string]any{"observed_at": at}}} - - if !w.news(quiet) { - t.Fatal("the first reconcile after the engine started did not say what the machine is") - } - w.said(quiet) - - for _, after := range []time.Duration{5 * time.Minute, 12 * time.Hour, 22*time.Hour + 55*time.Minute} { - at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(after) - if w.news(quiet) { - t.Errorf("%s after the mesh heard it, an unchanged reconcile was said again", after) - } - } - // The reconcile that falls past 23 hours says it, so it reaches the mesh inside the day. - at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(23*time.Hour + 5*time.Minute) - if !w.news(quiet) { - t.Fatal("a day on, the reconcile did not say what the machine is") - } - w.said(quiet) - at = at.Add(5 * time.Minute) - if w.news(quiet) { - t.Error("once said, it was said again at the next reconcile") - } -} - -// A delivery's account carries the profile too: heard, it starts the day again. -func TestADeliveryHeardStartsTheDayAgain(t *testing.T) { - at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) - w := &adoptionWatch{now: func() time.Time { return at }} - report := link.Report{Outward: []string{"eth0"}, Profile: map[string]any{"capabilities": []any{}}} - w.said(report) - at = at.Add(20 * time.Hour) - w.said(report) // a delivery's apply - at = at.Add(20 * time.Hour) - if w.news(report) { - t.Error("said again unasked 20 hours after a delivery told the mesh") - } -} - -// A refusal carries nothing about the machine, so it is never said for being a day old. -func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) { - at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) - w := &adoptionWatch{now: func() time.Time { return at }} - if w.machineDue(link.Report{Refused: "no"}) { - t.Error("a refusal was due for what the machine is made of") - } -} - -// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic -// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses -// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first -// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no -// monotonic reading, and judge the day as the wall clock says it. -func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) { - before := time.Now() // carries a monotonic reading - asleep := before.Add(3 * time.Second) - if asleep.Sub(before) != 3*time.Second { - t.Fatal("the monotonic reading is not what this test simulates with") - } - calls := 0 - w := &adoptionWatch{now: func() time.Time { - calls++ - if calls == 1 { - return before - } - // The wall clock a day on, as a machine's clock reads after waking. - return asleep.Round(0).Add(24 * time.Hour) - }} - report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}} - w.said(report) - if strings.Contains(w.machineSaid.String(), "m=") { - t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid) - } - if !w.news(report) { - t.Error("a day of wall time on, a laptop that slept did not say what it is") - } -} - -// And the watch's own clock never carries a monotonic reading. -func TestTheWatchClockIsWallTime(t *testing.T) { - w := &adoptionWatch{} - if s := w.clock().String(); strings.Contains(s, "m=") { - t.Errorf("the watch's clock carries a monotonic reading: %s", s) - } - w = &adoptionWatch{now: time.Now} - if s := w.clock().String(); strings.Contains(s, "m=") { - t.Errorf("a given clock's time kept its monotonic reading: %s", s) - } -} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index a0c4e1b..eb54e8b 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -31,6 +31,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/accounts" + "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/bundle" "github.com/novox/mesh-host/internal/firewall" @@ -104,9 +105,6 @@ reach, run as root at the machine. profile what this machine can be asked to do inventory what this machine is, and what it holds - apply FILE make this machine match a declaration from a file - reconcile make this machine match what the mesh last told it — or, before any - mesh has, the bundle this host carries bundle show what this host carries key make this machine's tunnel key, or read the one it made, and print the public half: what its join token is issued for (novox/hq ADR 0169) @@ -126,10 +124,9 @@ reach, run as root at the machine. --state where this node keeps what it knows (default /var/lib/mesh-host/state.json) --dry-run say what applying would change, and change nothing -Both apply and reconcile say what they would change before changing anything, and refuse a -declaration for the other mode than this node is in, or one older than what the mesh last said. - -It connects to nothing and listens on nothing. What it applies comes from a file. +Apply says what it would change before changing anything, and refuses a declaration for the other mode than this +node is in. It is for a machine the mesh has not spoken to. Once enrolled, the node follows its wants (novox/hq ADR +0539): it applies a module from the composition its want names, and no whole declaration. ` func main() { @@ -207,13 +204,6 @@ func parseArgs(args []string) (string, options, error) { rest = rest[1:] } - if command == "apply" { - if len(positionals) != 1 { - return "", opts, errors.New("apply needs exactly one declaration file") - } - opts.file = positionals[0] - return command, opts, nil - } if command == "hand-over" { if len(positionals) != 1 { return "", opts, errors.New("hand-over : the path of one directory used as found") @@ -270,29 +260,6 @@ func run(ctx context.Context, command string, opts options) error { writeInventory(inv) return nil - case "apply": - raw, err := os.ReadFile(opts.file) - if err != nil { - return fmt.Errorf("reading the declaration: %w", err) - } - // ParseTrusted: a file handed to the host by someone already running it as root is - // not the link. novox/hq ADR 0005 bounds what a REMOTE party may push; someone who - // can write this file and run this binary can do anything the binary can, so refusing - // them an action would buy nothing and would make an action untestable except by - // rebuilding the bundle. - d, err := declaration.ParseFileTrusted(raw) - if err != nil { - return err - } - return runApply(ctx, opts, d, raw, fromFile) - - case "reconcile": - d, raw, from, err := reconcileSource(opts) - if err != nil { - return err - } - return runApply(ctx, opts, d, raw, from) - case "bundle": if bundle.IsEmpty(builtFor) { fmt.Println("this host carries no bundle") @@ -412,58 +379,12 @@ type provenance int const ( // fromFile is `apply FILE`: handed to the host by someone already running it as root. fromFile provenance = iota - // fromBundle is `reconcile` on a machine the mesh has told nothing yet: the bundle carried in + // fromBundle is genesis on a machine the mesh has told nothing yet: the bundle carried in // the binary, the first node's path before its mesh is up (novox/hq ADR 0004). fromBundle - // fromDeclared is `reconcile` on a node the mesh has spoken to: what it last said, kept - // signed beside the state (declared.json), verified again before it is applied. - fromDeclared ) -// reconcileSource is which declaration `reconcile` holds this machine to. -// -// **What the mesh last said, never the bundle, once the mesh has said anything** (novox/hq issue -// 104). The bundle is right at genesis and stale a minute later — genesis rewrites it for the -// machine before applying it, and every declaration since came from the controller — and on an -// adopted node it is a converged declaration for a machine that is not converged. A node that -// has been told something and cannot prove it is the mesh's is refused, with the reason; the -// bundle is not applied in its place. -func reconcileSource(opts options) (*declaration.Declaration, []byte, provenance, error) { - path := store.DeclaredPath(opts.state) - _, err := store.ReadDeclared(path) - switch { - case err == nil: - mine, err := identity.Load(identity.Path(opts.state)) - if err != nil { - return nil, nil, 0, fmt.Errorf("this node was told a declaration by the mesh, kept at %s, "+ - "and cannot prove it is the mesh's: %w\n\nIt is not applied unproven, and the bundle "+ - "this host carries is not applied in its place: that was consumed at genesis", path, err) - } - raw, err := store.LoadDeclared(path, mine.Membership.Signer) - if err != nil { - return nil, nil, 0, fmt.Errorf("%w\n\nThe bundle this host carries is not applied in its "+ - "place: that was consumed at genesis", err) - } - d, err := declaration.Parse(raw) - if err != nil { - return nil, nil, 0, err - } - return d, raw, fromDeclared, nil - case errors.Is(err, store.ErrNothingDeclared): - d, err := bundle.Load(builtFor) - if err != nil { - return nil, nil, 0, err - } - return d, bundle.Raw(builtFor), fromBundle, nil - default: - return nil, nil, 0, err - } -} - func (p provenance) origin() string { - if p == fromDeclared { - return store.OriginDeclared - } // A file handed to the host is applied as the bundle is: what it puts here is invisible to // the removal pass of a declaration that later arrives from the mesh (novox/hq issue 010). return store.OriginCarried @@ -473,8 +394,6 @@ func (p provenance) name(opts options) string { switch p { case fromBundle: return "the carried bundle" - case fromDeclared: - return "what the mesh last told this node (" + store.DeclaredName + ")" } return opts.file } @@ -499,10 +418,8 @@ func short(digest string) string { // them again — and everything the mesh declared read as no longer declared and removed. Even the // very declaration the mesh last sent, applied from a file, would plan to remove the foundation. // `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so. -func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, order link.Order) error { +func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error { switch from { - case fromDeclared: - return refuseOlder(kept, keptErr, order) case fromBundle: if known.Genesis == nil || known.Genesis.Digest == digest { return nil @@ -536,8 +453,8 @@ func refuseStale(known store.State, kept store.Declared, keptErr error, digest s return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+ "node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+ "recorded as this machine's own, which the mesh could then never remove, and what the mesh "+ - "declared read as no longer declared — so no file is applied here: `reconcile` applies what "+ - "the mesh last said, and `push` from the controller changes it", + "declared read as no longer declared — so no file is applied here: the node follows its wants "+ + "(novox/hq ADR 0539)", short(last), store.DeclaredName, what) } @@ -585,13 +502,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the // note, or to the kept declaration when the note predates it. kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)) - if from == fromDeclared { - if known.Mode != "" && known.Mode != apply.ModeOf(d) { - fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n", - known.Mode, apply.ModeOf(d)) - } - known.Mode = apply.ModeOf(d) - } else if known.Mode == "" && keptErr == nil { + if known.Mode == "" && keptErr == nil { if last, err := declaration.Parse(kept.Declaration); err == nil { known.Mode = apply.ModeOf(last) } @@ -599,14 +510,9 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw if err := apply.CheckMode(known, d); err != nil { return err } - if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil { + if err := refuseStale(known, kept, keptErr, digest, from); err != nil { return err } - // The generation, as the link's deliveries are held to it (novox/hq issue 234): one composed from an - // older view of the assignments than this machine applied would undeclare what the mesh still assigns. - if refused := olderGeneration(known, d); refused != nil { - return errors.New(olderGenerationWords(*refused)) - } // A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has // no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty. @@ -674,7 +580,6 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // What this apply settles about the node, whichever way it went. The mode is what the // declaration said and the check above agreed with; the bundle, once applied, is consumed. updated.Mode = apply.ModeOf(d) - updated.Generation = appliedGeneration(known, d) // raised, never lowered (novox/hq issue 234) if from == fromBundle { updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()} } @@ -1179,8 +1084,9 @@ func runLink(ctx context.Context, opts options) error { Signer: mine.Membership.Signer, } - applier := func(ctx context.Context, raw, signature []byte) link.Report { - report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say) + // What follows an apply of a module (the node agent's, novox/hq ADR 0539): a membership it delivered for another bus + // is adopted, and a newer node-engine it placed makes this one stand aside. + afterApply := func() { // **A declaration may carry this machine's membership for another bus.** It arrives as a // sealed file like any secret, and is read after the rest has applied so the bus it names is // standing before this machine leaves the one it is on (novox/hq design 28, task 5.2). @@ -1201,35 +1107,16 @@ func runLink(ctx context.Context, opts options) error { stoodAside = true standAside() } - return report } - // Two things at once, and the second is what makes disconnection ordinary. The link brings - // new declarations; the reconcile holds the machine in the last one whether the link is up or - // not. A laptop shut for a week comes back and reconciles — it does not come back and ask what - // it is (novox/hq ADR 0004). - // - // **Both only enqueue** (novox/hq to-be 45 §6). One worker applies, the newest declaration held - // when it starts, once, and makes one report; a reconcile due while a delivery waits is that - // delivery's apply. Reports a reconcile has to make unasked — what an adopted node holds changed, - // its firewall, its outward links — are said by the same worker, in the order they are made - // (novox/hq ADR 0100, issue 267). - watch := &adoptionWatch{} + // The node follows its wants; it is never sent a declaration (novox/hq ADR 0539 §9). What the session does is hold + // the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the + // controller asks. proof := &proving{statePath: opts.state, version: runningVersion(), say: say} - queue := &link.Queue{ + queue := &link.Session{ Membership: membership, - Apply: applier, - Reconcile: reconcileKept(opts, mine.Membership.Signer, sched, say), - News: watch.news, - // Counted as said only once the broker has taken it: queued and lost — the link down, the - // publish refused — the change would never be said again (novox/hq ADR 0100). - // And the first account under this build is what proves it to the launcher (to-be 45 §8). - Heard: func(r link.Report) { - watch.said(r) - proof.heard(r) - }, - Unsaid: unsaidBeside(opts.state), - Numbers: numbersBeside(opts.state), + // And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8). + HaveWritten: proof.haveWritten, // And the operator's hand-over of a directory used as found, carried by the controller from the // terminal (novox/hq issue 356): judged and recorded here, applied by the next apply. HandOver: handOverFrom(opts.state), @@ -1248,6 +1135,15 @@ func runLink(ctx context.Context, opts options) error { Say: say, Timeout: opts.timeout, } + current := &agentNow{} + setRunsHere(runsFromAgent(current)) + factsOfNow := func(ctx context.Context, ag *agent.Agent) { + sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag) + } + queue.OnLink = func(ctx context.Context) { + factsOfNow(ctx, current.get()) + afterApply() + } // And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3, // issue 676): the vault is asked over this queue's link, so the queue is named once it exists. queue.MoveSecret = moveSecretFrom(opts.state, queue) @@ -1258,25 +1154,6 @@ func runLink(ctx context.Context, opts options) error { }, func(verb string, args any) (json.RawMessage, error) { return queue.AskVault(ctx, verb, args) }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo) - worker := make(chan struct{}) - go func() { - defer close(worker) - queue.Run(aside) - }() - // **A host starting is a reason to reconcile** (to-be 45 §6: the self-update hand-over is one of - // the four): its scheduled steps are armed from the declaration the node kept by the first apply, - // and a successor that waited five minutes for it left them unarmed for five. - // - // **A host on trial says its account whether or not it is news** (to-be 45 §8): the launcher - // waits for this build to report its declaration, and a converged machine with nothing new to - // say would otherwise not say anything until the mesh next sent it something. - if unproved(opts.state, runningVersion()) { - queue.ReportAsked() - } else { - queue.ReconcileDue() - } - go holdTheMachine(ctx, queue) - go busyPasses(ctx, queue, BusyAskEvery) // And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq // ADR 0240) — between applies, which is when a container crash-loops. if judging != nil { @@ -1291,6 +1168,14 @@ func runLink(ctx context.Context, opts options) error { go judging.Probe(aside) go judgeWhatRuns(aside, judging, queue, say) } + // The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the + // busy containers are asked for again soon (ADR 0360 §11). + go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow) + go busyPasses(aside, func() { + if ag := current.get(); ag != nil { + ag.Retry(busyModules()...) + } + }, BusyAskEvery) // And the core builds this host placed are judged, whichever host placed them (to-be 45 §8). go watchWhatThisHostPlaced(aside, mine.Node, queue, say) // And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller @@ -1301,9 +1186,6 @@ func runLink(ctx context.Context, opts options) error { go serveFilling(aside, opts, mine.Membership.Signer, say) held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx)) - // The act in hand finishes and is kept before this process exits: an apply that stood aside with - // no link open has its report only in the unsaid store, and only once the worker has written it. - <-worker // **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back // before it has run once. The context this returns on was cancelled deliberately, so its error // is not a fault to report. @@ -1313,241 +1195,6 @@ func runLink(ctx context.Context, opts options) error { return held } -// unsaidFile keeps the last apply's report beside the node's state until the mesh has taken it -// (novox/hq issue 264), so a report lost when this host stood aside for its successor — or died -// between applying and publishing — is said by whichever host runs next, once it is linked. -type unsaidFile struct{ path string } - -func unsaidBeside(statePath string) unsaidFile { return unsaidFile{path: store.UnsaidPath(statePath)} } - -func (u unsaidFile) Keep(r link.Report) error { - // A report naming no declaration is one the mesh cannot match to what it sent, and keeping it - // would only re-say an older apply after a newer one was refused. - if r.Declared == "" { - return store.ClearUnsaid(u.path) - } - body, err := json.Marshal(r) - if err != nil { - return err - } - return store.SaveUnsaid(u.path, body) -} - -func (u unsaidFile) Said(declared string) error { - kept, ok, err := u.Pending() - if err != nil || !ok || kept.Declared != declared { - return err - } - return store.ClearUnsaid(u.path) -} - -func (u unsaidFile) Pending() (link.Report, bool, error) { - raw, err := store.ReadUnsaid(u.path) - if err != nil || raw == nil { - return link.Report{}, false, err - } - var r link.Report - if err := json.Unmarshal(raw, &r); err != nil { - // Unreadable is forgotten rather than kept for ever: it can never be said. - _ = store.ClearUnsaid(u.path) - return link.Report{}, false, fmt.Errorf("the kept report at %s is unreadable, and is dropped: %w", u.path, err) - } - return r, r.Declared != "", nil -} - -// numbersFile keeps the queue's report sequence and its count of declarations refused as older beside -// the node's state (novox/hq to-be 45 §6), where a successor reads them and goes on from there. -type numbersFile struct{ path string } - -func numbersBeside(statePath string) numbersFile { - return numbersFile{path: store.NumbersPath(statePath)} -} - -func (n numbersFile) Read() (int64, int64, error) { - kept, err := store.ReadNumbers(n.path) - return kept.ReportSequence, kept.RefusedOlder, err -} - -func (n numbersFile) Save(reportSequence, refusedOlder int64) error { - return store.SaveNumbers(n.path, store.Numbers{ReportSequence: reportSequence, RefusedOlder: refusedOlder}) -} - -// adoptionWatch remembers what the node last said about what it holds and its firewall, so a -// reconcile speaks unasked only when that changed. -type adoptionWatch struct { - mu sync.Mutex - last string - // faulty is whether the last report the mesh heard said an apply refused or failed (novox/hq issue 233). - faulty bool - // machineSaid is when the mesh last heard what this machine is made of (novox/hq issue 160), and now the - // clock it is judged by (time.Now when nil). - machineSaid time.Time - now func() time.Time -} - -// machineEvery is how old what a machine said of itself may grow before a reconcile says it again unasked -// (novox/hq issue 160): a machine nobody pushed to still says it once a day, so no fact the mesh holds of it -// is older than a day. Under a day by more than one reconcile interval, so the reconcile that says it falls -// inside the day rather than just past it. -const machineEvery = 23 * time.Hour - -// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's -// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept -// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day. -func (w *adoptionWatch) clock() time.Time { - if w.now != nil { - return w.now().Round(0) - } - return time.Now().Round(0) -} - -// machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago -// or more. A refusal carries none, so it never is. -func (w *adoptionWatch) machineDue(r link.Report) bool { - return r.Refused == "" && len(r.Profile) > 0 && w.clock().Sub(w.machineSaid) >= machineEvery -} - -// fingerprint is what a report says about adoption: each hold and whether it changed, the -// firewall, and what is reachable on the machine — which only an adopted node reports, and which -// is what the controller previews a flip from, so a port that opens or closes between deliveries -// must reach it too (novox/hq ADR 0100). -func adoptionFingerprint(r link.Report) string { - // **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR - // 0140). The mesh composes no filter for a machine that has not said, so a machine whose links - // changed — or which has only just learnt to say — has to say so without being asked. Left out, - // it could only speak when a declaration arrived, and a declaration cannot be composed until it - // has spoken: a machine waiting for a push that is waiting for the machine. - parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")} - // **And the networks those links are on** (novox/hq ADR 0340 §7): a port reached from the local - // network is admitted from them, so a laptop that joins another network, or a lease that moves, is - // said at the next reconcile rather than at the next push or the next day. - if networks := attachedNetworksOf(r); networks != "" { - parts = append(parts, "networks="+networks) - } - // And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk - // waiting on this machine starts its gate at the report that says it replaced. Not its output, which - // may change on every ask. - for _, b := range r.Busy { - parts = append(parts, "busy "+b.ID+"="+b.State) - } - for _, h := range r.Held { - parts = append(parts, "held "+h.ID+"="+h.Changed) - } - // And what filters the machine, with the found firewall's state (novox/hq ADR 0168): a rule the - // operator removes between declarations, or a front end enabled again, is said at the next - // reconcile rather than at the next push. - for _, f := range r.Filters { - parts = append(parts, "filter "+f.Owner+" "+f.Where+" "+f.Refuses) - } - // And a witness's verdict (to-be 45 §8): one reached or one ended is said at the next reconcile. - for _, v := range r.Rollbacks { - parts = append(parts, fmt.Sprintf("rollback %s %s %s %s", v.Component, v.From, v.To, v.Outcome)) - } - // **And how the apply went** (novox/hq issue 233): refused, clean, or which resources it did not apply. A - // reconcile is the only apply a machine makes between deliveries, so a machine whose package manager stopped - // answering, or answers again, says so at the next reconcile — otherwise the mesh's record of what it did - // stays whatever the last delivery left, and a fault that began or ended between deliveries is never seen. - // Not the words: an error that carries a time would read as new on every reconcile. - parts = append(parts, applyState(r)) - if r.FoundFirewall != nil { - parts = append(parts, fmt.Sprintf("found-firewall %s active=%v retired-by=%s", r.FoundFirewall.Kind, - r.FoundFirewall.Active, r.FoundFirewall.RetiredBy)) - } - for _, reach := range r.Reachable { - parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address, - reach.Port, reach.By, reach.Published, reach.ContainerPort)) - } - sort.Strings(parts[2:]) - return strings.Join(parts, "\n") -} - -// attachedNetworksOf is the networks a report's profile says the machine is attached to, as one line: the -// part of what the machine is made of that the filter is composed from (novox/hq ADR 0340). -func attachedNetworksOf(r link.Report) string { - machine, ok := r.Profile["machine"].(map[string]any) - if !ok { - return "" - } - networks, ok := machine["networks"].([]any) - if !ok { - return "" - } - said := make([]string, 0, len(networks)) - for _, n := range networks { - if entry, ok := n.(map[string]any); ok { - said = append(said, fmt.Sprintf("%v %v", entry["link"], entry["network"])) - } - } - return strings.Join(said, ",") -} - -// applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not -// apply and which keys its failure is said under. -func applyState(r link.Report) string { - switch { - case r.Refused != "": - return "apply refused" - case len(r.Failed) == 0: - return "apply clean" - } - keys := make([]string, 0, len(r.Failed)) - for k := range r.Failed { - keys = append(keys, k) - } - sort.Strings(keys) - return "apply failed " + strings.Join(keys, ",") + " unapplied " + strings.Join(r.Unapplied, ",") -} - -// faulty is whether a report says an apply refused or failed. -func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 } - -// news is whether a reconcile's report is worth saying unasked: what the machine is made of last reached the -// mesh a day ago or more (novox/hq issue 160); or it says something the last one the mesh heard -// did not, and it is about the machine (worthSaying) — or about how the apply went, refused or failed now, or -// clean after the last one the mesh heard was not (novox/hq issue 233). A refusal says nothing about the -// machine, but that the machine refuses is what the mesh raises a condition from, and a clean apply after it is -// what clears one. -func (w *adoptionWatch) news(r link.Report) bool { - w.mu.Lock() - defer w.mu.Unlock() - // What the machine is made of, once a day whatever else changed (novox/hq issue 160). - if w.machineDue(r) { - return true - } - if adoptionFingerprint(r) == w.last { - return false - } - return worthSaying(r) || faulty(r) || w.faulty -} - -// differs says whether a report says anything the last one that went out did not. It records -// nothing: what was said is what reached the mesh, not what was written down to send. -func (w *adoptionWatch) differs(r link.Report) bool { - w.mu.Lock() - defer w.mu.Unlock() - return adoptionFingerprint(r) != w.last -} - -// said records a report the mesh has actually been told. -func (w *adoptionWatch) said(r link.Report) { - w.mu.Lock() - defer w.mu.Unlock() - w.last = adoptionFingerprint(r) - w.faulty = faulty(r) - if len(r.Profile) > 0 { - w.machineSaid = w.clock() - } -} - -// changed is differs and said together, for a report published as it is made. -func (w *adoptionWatch) changed(r link.Report) bool { - if !w.differs(r) { - return false - } - w.said(r) - return true -} - // rousedBySignal is the machine telling this process that its link is probably stale. // // **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would @@ -1586,14 +1233,6 @@ func rousedBySignal(ctx context.Context) link.Roused { return out } -// ReconcileEvery is how often a node re-applies what it was last told. -// -// Not driven by the link. Changes are pushed, so this is not polling for them — it is the answer -// to a machine drifting: a file edited by hand, a container stopped by somebody, a service that -// died. A node that only acted when told would hold its state exactly until something else -// changed it, and then for ever. -const ReconcileEvery = 5 * time.Minute - // judging is the serving host's liveness judge (novox/hq ADR 0240); nil in a one-shot command, which // reports no health, and in a test. var judging *liveness.Judge @@ -1690,7 +1329,7 @@ const ( // the bus when a state changed, again every minute while one is not healthy, and every five minutes // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it. -func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) { +func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) { ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() var lastSaid time.Time @@ -1911,34 +1550,34 @@ func orNothing(s, unless string) string { return s } -// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq -// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is -// waiting is that delivery's apply. -func holdTheMachine(ctx context.Context, queue *link.Queue) { - ticker := time.NewTicker(ReconcileEvery) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - queue.ReconcileDue() - } - } -} - // busyWaiting is whether the last apply left a container running because it said busy (novox/hq ADR 0335). var busyWaiting atomic.Bool +// busyBy is, per module, whether its last apply left a container running because it said busy; busyWaiting is any of them. +var ( + busyMu sync.Mutex + busyBy = map[string]bool{} +) + +func noteBusy(module string, waiting bool) { + busyMu.Lock() + defer busyMu.Unlock() + busyBy[module] = waiting + any := false + for _, w := range busyBy { + any = any || w + } + busyWaiting.Store(any) +} + // BusyAskEvery is how often a container waiting because it said busy is asked again (novox/hq ADR 0335 §2): // each ask is a pass of its own, so a build that ends is followed by its replacement within this, not // within the reconcile's five minutes — and the bound of 15 minutes is met within this of its end. const BusyAskEvery = 20 * time.Second -// busyPasses asks for a pass every so often while a container waits because it said busy. **It asks; it -// does not apply**, as holdTheMachine: the queue's worker does, when its turn comes, so a pass never holds -// the machine for the wait, and a delivery waiting is that pass's apply. -func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) { +// busyPasses asks for a pass every so often while a container waits because it said busy (ADR 0360 §11). **It asks; it +// does not apply**: the node agent does, when its turn comes, so a pass never holds the machine for the wait. +func busyPasses(ctx context.Context, ask func(), every time.Duration) { ticker := time.NewTicker(every) defer ticker.Stop() for { @@ -1947,66 +1586,12 @@ func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) { return case <-ticker.C: if busyWaiting.Load() { - queue.ReconcileDue() + ask() } } } } -// reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it -// was last told, read once it is this act's turn, and say what did not go. -// -// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or its firewall -// changed, because that is how a predecessor still writing is caught (novox/hq ADR 0100); and on any -// node when it can say which links face outside (ADR 0140) — a converged node holds nothing and found -// no firewall, so without that it could speak only in reply to a declaration, while the mesh composes -// no declaration for a node that has not said which links face outside. Whether a report is news is -// the queue's News (the watch's news), so an unchanged answer costs one comparison every reconcile and -// nothing on the bus. And on any node when how the apply went changed — refused, clean, or which resources it -// did not apply — since that is what the mesh raises a machine that does not apply from, and clears it by -// (novox/hq issue 233). -func reconcileKept(opts options, signer ed25519.PublicKey, sched *apply.Scheduler, - say link.Announce) link.Reconcile { - return func(ctx context.Context) (link.Report, bool) { - report, err := reapplyKept(ctx, opts, signer, sched, say) - if errors.Is(err, store.ErrNothingDeclared) { - // Nothing to hold this machine to yet. Ordinary on a node that has enrolled and not - // been assigned anything. - return link.Report{}, false - } - if err != nil { - say("cannot re-apply what this node was told: " + err.Error()) - return link.Report{}, false - } - switch { - case report.Refused != "": - say("what this node was last told no longer applies: " + report.Refused) - case len(report.Failed) > 0: - say(fmt.Sprintf("holding this machine: %d applied, and %v", len(report.Applied), report.Failed)) - } - return report, true - } -} - -// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked. -// -// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline -// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a -// converged node — which holds nothing and found no firewall — could speak only in reply to a -// declaration, while the mesh composes no declaration for a node that has not said which links face -// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them. -// -// A refused report says nothing about the machine, so it is not news; the refusal is said on the -// console where a person reads it. -func worthSaying(report link.Report) bool { - if report.Refused != "" { - return false - } - return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0 || - len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0 || - len(report.Rollbacks) > 0 || len(report.Busy) > 0 -} - // announceOr is what the apply writes its detail with, given what the caller has to say things with. // // **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply @@ -2033,148 +1618,86 @@ func announceOr(say link.Announce) func(string) { // run by hand beside this service — the lock beside the state does the same (store.Lock). var applying sync.Mutex -// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can -// go on obeying it while disconnected. One at a time, whoever asks. Given unsigned, the bytes are -// what this node kept, already verified against the mesh's key on load. -func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared, - sched *apply.Scheduler, say link.Announce) link.Report { - applying.Lock() - defer applying.Unlock() - return applyAndKeepHeld(ctx, opts, raw, signed, sched, say) +// appliedModule is how the apply of one module's declaration came out. +type appliedModule struct { + // Refused is why nothing was applied, when nothing was. + Refused string + Outcome apply.Report + Updated store.State + // Err is the apply's own error, when it failed part way. + Err error + Probe *unanswered } -// reapplyKept re-applies what this node was last told, **read once it is this apply's turn**. -// -// Read before waiting, it was whatever had been kept when the timer fired — and a declaration -// the link was applying at that moment is kept only once its apply ends. The reconcile then waited -// for that apply and applied the older declaration over it: a module assigned a second earlier was -// given back on the spot, and the report it sent named a declaration the mesh no longer recorded -// as sent, so a plan waiting on this machine waited for a report that could not come (novox/hq -// issues 257, 261). Read under the lock, it is always the latest the mesh said. -func reapplyKept(ctx context.Context, opts options, signer ed25519.PublicKey, - sched *apply.Scheduler, say link.Announce) (link.Report, error) { +// applyModuleHeld applies one module's declaration (novox/hq ADR 0539 §9): the resources of its composition, with the +// compositions the node's other modules run held out of any removal. A nil scope is a whole declaration, as the carried +// bundle is. One at a time, and the state is read and saved under the lock of the node's state. +func applyModuleHeld(ctx context.Context, opts options, declared *declaration.Declaration, scope *apply.Scope, + sched *apply.Scheduler, say link.Announce) appliedModule { applying.Lock() defer applying.Unlock() - declared, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) - if err != nil { - return link.Report{}, err - } - return applyAndKeepHeld(ctx, opts, declared, nil, sched, say), nil -} - -// applyAndKeepHeld is applyAndKeep for a caller already holding applying. -func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *store.Declared, - sched *apply.Scheduler, say link.Announce) link.Report { unlock, err := store.Lock(opts.state, nil) if err != nil { - return link.Report{Refused: err.Error()} + return appliedModule{Refused: err.Error()} } defer unlock() - declared, err := declaration.Parse(raw) - if err != nil { - return link.Report{Refused: err.Error()} - } - order := link.Order{Epoch: declared.Epoch, Sequence: declared.Sequence} - - // **Older than what this node applied is refused, before anything is read from the machine** - // (novox/hq to-be 45 §6, ADR 0227 rule 2): a controller that lost its lease and goes on sending. - // Read under the lock, so what it is compared with is what the last apply kept. Only a delivery: - // what a reconcile applies is what was kept. - if signed != nil { - kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)) - if held, older := olderThanKept(kept, keptErr, order); older { - return link.Report{Declared: digestOf(raw), Order: order, OlderThan: &held, - Refused: fmt.Sprintf("this declaration was sent under lease epoch %d, sequence %d, and this "+ - "node has applied epoch %d, sequence %d — from the controller holding the lease now. "+ - "Refused whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence)} - } - } - known, err := store.Load(opts.state) if err != nil { - return link.Report{Refused: err.Error()} + return appliedModule{Refused: err.Error()} } - // **Composed from an older view of the assignments than this machine applied is refused, before - // anything is read from the machine** (novox/hq issue 234). A newer sequence carrying an older view - // undeclared four assigned modules on 2026-10-04; the sequence cannot tell it, the generation can. - // Only a delivery: what a reconcile applies is what was kept. - if signed != nil { - if refused := olderGeneration(known, declared); refused != nil { - return link.Report{Declared: digestOf(raw), Order: order, OlderGeneration: refused, - Refused: olderGenerationWords(*refused)} - } - } - // A declaration from the mesh is the controller's word on the node's mode — the flip arrives - // as exactly that, the first converged declaration after adopted ones — and becomes the - // record. So does what this node kept: it is signed by the mesh and verified on load, where - // the state's note of the mode is this host's own. Where they disagree the note is wrong — a - // genesis re-run over a node the mesh converged since, a state saved when the kept declaration - // could not be — and it is repaired and said, not obeyed: refusing would hold this node off what - // the mesh said until a delivery that comes only when something changes (novox/hq issue 104). - if signed == nil && known.Mode != "" && known.Mode != apply.ModeOf(declared) { - if say != nil { - say(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired", - known.Mode, apply.ModeOf(declared))) - } + node := scope == nil || scope.Module == store.NodeModule + // The node's own composition is the controller's word on the node's mode — the flip arrives as exactly that, the + // first converged composition after adopted ones — and becomes the record (novox/hq issue 104). + if node && known.Mode != "" && known.Mode != apply.ModeOf(declared) { + announceOr(say)(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired", + known.Mode, apply.ModeOf(declared))) known.Mode = apply.ModeOf(declared) } - // **A file given by digest is fetched and verified before anything of the declaration is applied** (novox/hq ADR - // 0486 §1, issue 1695): one object missing or differing refuses the declaration whole. The signed bytes are what is - // kept; only what is applied has the content in place of the reference. + // **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR + // 0486 §1, issue 1695): one object missing or differing refuses the module whole. if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil { - return link.Report{Declared: digestOf(raw), Order: order, - Refused: "a file of this declaration could not be had by its digest: " + err.Error()} + return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()} } built, err := system.For(builtFor) if err != nil { - return link.Report{Refused: err.Error()} + return appliedModule{Refused: err.Error()} } if err := system.Check(built, declared); err != nil { - return link.Report{Refused: err.Error()} + return appliedModule{Refused: err.Error()} } - // A package manager that does not answer fails only what needs it, and the report says so (novox/hq - // issue 233, probe.go). + // A package manager that does not answer fails only what needs it, and the report says so (novox/hq issue 233). probe, err := probePackageManager(ctx, built) if err != nil { - return link.Report{Refused: err.Error()} + return appliedModule{Refused: err.Error()} } if probe != nil { announceOr(say)(probe.said()) } - // Declared, not carried. A declaration from the mesh removes only what the mesh previously - // declared — never what this machine raised for itself from its bundle (04-ISSUES/010). - // **What the apply says goes to the console, which is the journal when this runs as a service.** - // - // It was nil, and nil is silence. The one-shot path has always passed a real one, so every detail - // the apply produces — a file held, a container replaced, the found firewall retired — was visible - // when a person ran it by hand and discarded in the way the host actually runs. Measured: after a - // machine was converged and its found firewall was not retired, what the host decided was - // unrecoverable, because it had said it to nobody (novox/hq 04-ISSUES/143). - // - // `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs - // no new mechanism — only for the argument to be passed. - // Once more when the package manager did not answer and this apply wrote something (issue 233, probe.go). + // **What the apply says goes to the console, which is the journal when this runs as a service** (novox/hq + // 04-ISSUES/143). Once more when the package manager did not answer and this apply wrote something (issue 233). outcome, updated, applyErr, probe, firstPass := applyMindingProbe(ctx, built, probe, known, func(known store.State) (apply.Report, store.State, error) { - return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared, + if scope == nil { + return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared, + apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)), + apply.WindowsIn(filepath.Dir(opts.state))) + } + return apply.ApplyModule(ctx, built, declared, *scope, known, store.OriginDeclared, apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state))) }, announceOr(say)) - // The mode the mesh said, recorded whichever way the apply went: the declaration is kept - // either way, and the node is held to it from the next reconcile (novox/hq ADR 0100). - updated.Mode = apply.ModeOf(declared) - // And the generation it came from, the same way, raised and never lowered (novox/hq issue 234). - updated.Generation = appliedGeneration(known, declared) + if node { + updated.Mode = apply.ModeOf(declared) + } - // Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's - // rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it - // did before it stopped. + // Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's rules: the search + // for setuid programs runs again (novox/hq issue 361). A failed apply counts what it did before it stopped. moved := append(append([]apply.Outcome(nil), firstPass...), outcome.Outcomes...) var partial *apply.Error if errors.As(applyErr, &partial) { @@ -2184,27 +1707,43 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto setuidSearch.SearchAgain() } - // Saved whichever way it went. Recording only on success would lose the footprint of a - // failed apply, and that footprint is on the machine either way. + // Saved whichever way it went: the footprint of a failed apply is on the machine either way. if saveErr := store.Save(opts.state, updated); saveErr != nil { - return link.Report{Refused: "applied, and the node's state could not be saved: " + - saveErr.Error()} + return appliedModule{Refused: "applied, and the node's state could not be saved: " + saveErr.Error()} } - // Re-establish the scheduled steps from the declaration just applied (novox/hq ADR 0053). Done - // from the parsed declaration, which is the source of truth (ADR 0018): a schedule newly declared - // is armed, one whose image, environment or cadence changed is re-armed, and one no longer - // declared is forgotten — and after a host restart the first apply rebuilds them all. A nil - // scheduler is the one-shot CLI path, which exits rather than staying up to fire anything. + // Re-establish the scheduled steps of this declaration (novox/hq ADR 0053): a schedule newly declared is armed, one + // whose image, environment or cadence changed is re-armed, one no longer declared is forgotten, and the other + // modules' are left as they are. A nil scheduler is a test. held := map[string]bool{} for _, h := range updated.Held { held[h.ID] = true } if sched != nil { - sched.Sync(declared, held) + if scope == nil { + sched.Sync(declared, held) + } else { + sched.SyncModule(declared, held, func(id string) bool { + m, ok := declaration.ModuleGuess(id) + return ok && m == scope.Module || (!ok && scope.Module == store.NodeModule) + }) + } } + name := "" + if scope != nil { + name = scope.Module + } + noteBusy(name, outcome.Waiting()) + return appliedModule{Outcome: outcome, Updated: updated, Err: applyErr, Probe: probe} +} - report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(), +// factsOf is what this node says of itself: its ports, its profile, how what runs here is, which links face outside, +// what it holds, its filters and, when adopted, its firewall and what is reachable. The facts are an event of the node +// to the controller (novox/hq ADR 0539 §9); declared is the declaration the node was last held to, updated its state, +// outcome what the last apply did (empty when nothing was applied), and held the ids it holds. +func factsOf(ctx context.Context, opts options, declared *declaration.Declaration, updated store.State, + outcome apply.Report, held map[string]bool) link.Report { + report := link.Report{Carried: carriedPorts(updated), Host: runningVersion(), Rollbacks: standingRollbacks(opts.state), Witness: link.WitnessContract, Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))} // What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just @@ -2326,36 +1865,31 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto } report.Applied = append(report.Applied, change.ID) } - // Kept whichever way it went, so a node that is disconnected next minute still knows what it - // was told. Saved after applying rather than before: what is kept is what this node acted on. - var before []string - if signed != nil { - // The modules the declaration kept until now held, so a have key written for one the new declaration no longer - // holds is written removed even by a node-engine that started since (issue 1369). - if kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)); keptErr == nil { - if old, parseErr := declaration.ParseTrusted(kept.Declaration); parseErr == nil { - for _, f := range moduleFacts(old, apply.Report{}, nil) { - before = append(before, f.Module) - } - } - } - if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil { - report.Failed = map[string]string{"keeping the declaration": err.Error()} - } - } - if applyErr != nil { - report.Failed = map[string]string{"apply": applyErr.Error()} - } - report.Failed = withUnanswered(report.Failed, probe) - // And how each module of the declaration came out, for the have keys written beside this report (novox/hq ADR 0429 - // §4, issue 1132): not on the wire. - report.Modules = moduleFacts(declared, outcome, applyErr) - report.Before = before - // Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire. - report.Unapplied = unapplied(declared, outcome, applyErr) return report } +// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs +// here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it +// has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's. +func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, say link.Announce, + ag *agent.Agent) { + known, err := store.Load(opts.state) + if err != nil { + say("cannot say this node's facts: " + err.Error()) + return + } + declared := &declaration.Declaration{} + if d, err := declaredNow(opts.state, signer); err == nil { + declared = d + } + held := map[string]bool{} + for _, h := range known.Held { + held[h.ID] = true + } + report := factsOf(ctx, opts, declared, known, apply.Report{}, held) + session.SayFacts(ctx, report) +} + // waitForEnrolment returns this node's identity, waiting for one if it has none. // // It applies the carried bundle first, if there is one, because that is what a first node does @@ -2488,92 +2022,6 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say os.Exit(0) } -// refuseOlder refuses a declaration from the mesh that is older than the one this node holds. -// -// **By sequence, not by arrival** (novox/hq 04-ISSUES/107). What the host kept is the last thing -// the mesh said, signed; a declaration whose sequence is lower was composed before it, whatever -// order they arrived in — a backlog drained after the node was away, or a broker that split a burst. -// Applying it would make the machine into something the mesh had already moved past, which is the -// incident of issue 104 by another door. -// -// Only when both sides claim an order. A declaration with no sequence is one an older controller -// sent, and one kept with no sequence is one this host received before it understood them; in either -// case there is no order to compare, and refusing on a guess would strand the node the moment the -// controller is older than the host. Equal is the same declaration again, which reconciling is for. -// -// **And by the lease epoch first** (novox/hq to-be 45 §6): a declaration sent under an older epoch than -// the one kept here is a controller that lost its lease, whatever its sequence — the same rule the -// link's deliveries are held to (olderThanKept). Where both claim an epoch it alone decides, because a -// new lease holder's sequence says nothing against the last one's. -func refuseOlder(kept store.Declared, keptErr error, order link.Order) error { - if held, older := olderThanKept(kept, keptErr, order); older { - return fmt.Errorf("this declaration was sent under lease epoch %d, sequence %d, and the one kept "+ - "here under epoch %d, sequence %d: a controller that no longer holds the lease sent it. Refused "+ - "whole; nothing was applied", order.Epoch, order.Sequence, held.Epoch, held.Sequence) - } - sequence := order.Sequence - if sequence == 0 || keptErr != nil { - return nil - } - last, err := declaration.ParseTrusted(kept.Declaration) - if err != nil || last.Sequence == 0 || (order.Epoch > 0 && last.Epoch > 0) { - return nil - } - if sequence < last.Sequence { - return fmt.Errorf("this declaration is older than what the mesh last said to this node: it "+ - "is sequence %d, and the one kept here is %d. It arrived late — a backlog, or a broker "+ - "that split a burst — and applying it would make this machine into something the mesh has "+ - "already moved past. Refused whole; nothing was applied", sequence, last.Sequence) - } - return nil -} - -// olderGeneration is the refusal of a declaration composed from an older assignment generation than -// this machine applied, nil when it is not one (novox/hq issue 234). -// -// Only when both claim a generation: a declaration without one is from a controller that claims none, -// and a machine that applied none has nothing to hold it to — refusing on a guess would strand every -// machine the moment the controller is older than the node-engine. Equal is not older: a push of the -// same assignments is the ordinary case. A put-back carries the generation of what it puts back and -// says so; a gate putting a machine back is the one sender that may say something older. -func olderGeneration(known store.State, d *declaration.Declaration) *link.GenerationRefusal { - if d.PutBack || d.Generation <= 0 || known.Generation <= 0 || d.Generation >= known.Generation { - return nil - } - return &link.GenerationRefusal{Generation: d.Generation, Applied: known.Generation} -} - -// olderGenerationWords is a generation refusal as the report and a person read it. -func olderGenerationWords(r link.GenerationRefusal) string { - return fmt.Sprintf("this declaration was composed from assignment generation %d, and this node applied "+ - "generation %d: it was composed from a view of the assignments the mesh has moved past, and applying it "+ - "would undeclare what the mesh still assigns here (novox/hq issue 234). Refused whole; nothing was "+ - "applied", r.Generation, r.Applied) -} - -// appliedGeneration is the highest generation applied once this declaration is: raised by it, never -// lowered — a put-back of an older generation leaves it where it was (novox/hq issue 234). -func appliedGeneration(known store.State, d *declaration.Declaration) int64 { - return max(known.Generation, d.Generation) -} - -// olderThanKept says whether a declaration of this order is older than the one this node kept — the -// last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept -// declaration that cannot be read claims no order: what it is compared with is unknown, and refusing -// on a guess would hold the machine off everything the mesh sends until a person looked. That kept -// file is read, and refused by name, by the next reconcile. -func olderThanKept(kept store.Declared, keptErr error, order link.Order) (link.Order, bool) { - if keptErr != nil || order.Epoch <= 0 { - return link.Order{}, false - } - last, err := declaration.ParseTrusted(kept.Declaration) - if err != nil { - return link.Order{}, false - } - held := link.Order{Epoch: last.Epoch, Sequence: last.Sequence} - return held, order.Older(held) -} - // profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a // report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161). func profileAsReported(detected profile.Profile) map[string]any { diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 1aaf8bf..dbe477b 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "crypto/ed25519" "encoding/json" "errors" "os" @@ -14,8 +13,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/bundle" - "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" @@ -98,26 +95,6 @@ func TestNoCommandIsNotAnError(t *testing.T) { } } -func TestApplyNeedsExactlyOneDeclaration(t *testing.T) { - // `apply` takes a file where every other command takes nothing, so the leftover-argument - // rule has an exception — and an exception is where a parser stops refusing things it - // should. Both directions are checked. - if _, _, err := parseArgs([]string{"apply"}); err == nil { - t.Error("apply with no file was accepted") - } - if _, _, err := parseArgs([]string{"apply", "a.json", "b.json"}); err == nil { - t.Error("apply with two files was accepted") - } - - command, opts, err := parseArgs([]string{"apply", "decl.json", "--dry-run"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if command != "apply" || opts.file != "decl.json" || !opts.dryRun { - t.Errorf("parsed as command=%q file=%q dry-run=%v", command, opts.file, opts.dryRun) - } -} - func TestTheStateHasADocumentedDefault(t *testing.T) { // A host that wrote its state somewhere unexpected would forget what it owns on the next // run, and then leave everything it had applied behind forever. @@ -135,93 +112,22 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) { // off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the // standard library stops at the first non-flag argument wherever that argument is. for _, args := range [][]string{ - {"apply", "decl.json", "--dry-run", "--json"}, - {"apply", "--dry-run", "decl.json", "--json"}, - {"apply", "--dry-run", "--json", "decl.json"}, + {"hand-over", "decl.json", "--dry-run", "--json"}, + {"hand-over", "--dry-run", "decl.json", "--json"}, + {"hand-over", "--dry-run", "--json", "decl.json"}, } { command, opts, err := parseArgs(args) if err != nil { t.Errorf("%v: unexpected error: %v", args, err) continue } - if command != "apply" || opts.file != "decl.json" || !opts.dryRun || !opts.json { + if command != "hand-over" || opts.file != "decl.json" || !opts.dryRun || !opts.json { t.Errorf("%v parsed as file=%q dry-run=%v json=%v", args, opts.file, opts.dryRun, opts.json) } } } -// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds -// or its firewall changed — which is how a predecessor still writing is caught, without a report -// every five minutes saying nothing new. -func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) { - w := &adoptionWatch{} - held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}} - if !w.changed(held) { - t.Fatal("the first report of a hold was not said") - } - again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}} - if w.changed(again) { - t.Error("the same holds in another order were said again") - } - rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}} - if !w.changed(rewritten) { - t.Error("a held file rewritten by something else was not said") - } - if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) { - t.Error("a changed firewall was not said") - } - // What filters the machine is part of it (novox/hq ADR 0168): a predecessor's chain removed by - // hand, or the found firewall enabled again, is said without being asked. - filtered := link.Report{Firewall: "none", Held: rewritten.Held, - Filters: []link.Filter{{Where: "chain HAL-MESH-ONLY (iptables-legacy)", Owner: "other", Refuses: "-j DROP"}}} - if !w.changed(filtered) { - t.Error("a filter appearing was not said") - } - if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) { - t.Error("a filter removed was not said") - } - if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held, FoundFirewall: &link.FoundFirewall{Kind: "ufw", Active: true}}) { - t.Error("the found firewall coming back was not said") - } - if !worthSaying(link.Report{Filters: filtered.Filters}) { - t.Error("a report carrying only what filters the machine is not worth saying") - } -} - -// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue -// tells the watch every report the broker took (link.Queue.Heard). -func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) { - w := &adoptionWatch{} - report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}} - heard := w.said - heard(report) - if w.changed(report) { - t.Error("a reconcile repeated what the mesh had just heard") - } -} - -func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) { - // The controller previews a flip from what the node last said is reachable; a port that opened - // since must reach it without waiting for the next delivery (novox/hq ADR 0100). - w := &adoptionWatch{} - before := link.Report{Firewall: "ufw", Reachable: []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} - if !w.changed(before) { - t.Fatal("the first report was not said") - } - reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} - if w.changed(reordered) { - t.Error("the same reachable set was said again") - } - opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable, - link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})} - if !w.changed(opened) { - t.Error("a newly published port was not said") - } -} - // Defends the node's own record: the link and the reconcile loop both apply, and each reads the // state, acts, and writes it back — so they must not run at the same time, or the last save loses // what the other recorded. @@ -270,24 +176,6 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { } } -// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued -// and lost — the link down when the reconcile spoke — it must be said again. -func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) { - w := &adoptionWatch{} - held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}} - if !w.differs(held) { - t.Fatal("the first report of a change was not new") - } - // The link was down: nothing published it, so nothing says it was said. - if !w.differs(held) { - t.Error("a change that never reached the mesh was counted as said") - } - w.said(held) - if w.differs(held) { - t.Error("a change the mesh was told was said again") - } -} - // Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at // the point of application, whichever command delivered it, naming both — an adopted control-node // once applied its converged genesis bundle and closed itself for forty-five minutes. @@ -468,125 +356,6 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) { } } -// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to -// that — never to the bundle the host carries, which genesis consumed. -func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) { - was := builtFor - builtFor = "arch" - t.Cleanup(func() { builtFor = was }) - opts := stateWithMode(t, store.ModeAdopted) - - controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`) - if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said, - Signature: ed25519.Sign(controllerPrivate, said)}); err != nil { - t.Fatal(err) - } - - // Told, and unable to prove by whom: refused, and the bundle is not applied in its place. - _, _, _, err = reconcileSource(opts) - if err == nil || !strings.Contains(err.Error(), "not applied in its place") { - t.Errorf("a node that cannot prove what it was told fell back to something: %v", err) - } - - mine, err := identity.Generate("workstation") - if err != nil { - t.Fatal(err) - } - mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0", - Signer: controllerPublic, Password: "issued"} - if err := identity.Save(identity.Path(opts.state), mine); err != nil { - t.Fatal(err) - } - d, raw, from, err := reconcileSource(opts) - if err != nil { - t.Fatal(err) - } - if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil { - t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw)) - } - - // And before the mesh has said anything: the bundle, as it always was. A test binary carries - // only the placeholder, and asking for it is what proves the path. - if err := os.Remove(store.DeclaredPath(opts.state)); err != nil { - t.Fatal(err) - } - _, _, _, err = reconcileSource(opts) - if !errors.Is(err, bundle.ErrEmpty) { - t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err) - } -} - -// **A machine says which links face outside without being asked.** -// -// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only -// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say -// could speak only when a declaration arrived — and a declaration cannot be composed until it has -// spoken. A machine waiting for a push that is waiting for the machine. -func TestANewOutwardLinkIsSaidUnasked(t *testing.T) { - w := &adoptionWatch{} - first := link.Report{Firewall: "none"} - if !w.differs(first) { - t.Fatal("the first report should differ from nothing") - } - w.said(first) - - // Only the links changed, and nothing about adoption. - learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}} - if !w.differs(learnt) { - t.Fatal("a machine that has just learnt which links face outside would never say so, " + - "and could then never be sent a filter") - } - w.said(learnt) - if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) { - t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile") - } - - // And a link that changes — a laptop moving from a cable to a radio — is said too, because the - // filter is written around the old one until it is. - if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) { - t.Fatal("a changed outward link is not said, so the filter stays written around the old one") - } -} - -// **A converged machine can say which links face outside, unasked.** -// -// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an -// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so -// it could speak only in reply to a declaration, and the mesh composes no declaration for a node -// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines -// sat silent while the control plane refused to send them a filter. -func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) { - // A converged node's reconcile: nothing held, no found firewall, and the links it can see. - converged := link.Report{Outward: []string{"eth0"}} - if !worthSaying(converged) { - t.Fatal("a converged machine cannot say which links face outside, so it can never be " + - "sent a filter — a machine waiting for a push that is waiting for the machine") - } - - // An adopted node's reasons still hold, because that is how a predecessor still writing is caught. - if !worthSaying(link.Report{Firewall: "ufw"}) { - t.Fatal("an adopted machine no longer says which firewall it found") - } - if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) { - t.Fatal("an adopted machine no longer says what it holds") - } - - // And a reconcile with nothing to say stays silent, or every machine speaks every five minutes - // about nothing. - if worthSaying(link.Report{}) { - t.Fatal("a reconcile with nothing to say speaks anyway") - } - - // A refused report says nothing about the machine; the refusal is for the console. - if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) { - t.Fatal("a refused report is offered as news about the machine") - } -} - // **A host running as a service says what its apply did.** // // The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path @@ -611,63 +380,3 @@ func TestTheApplysLogIsNeverNil(t *testing.T) { t.Fatalf("the apply's detail did not reach the caller's announce: %v", said) } } - -// Defends novox/hq issues 257 and 261: a reconcile due while the link applies a newer declaration -// applies that newer one once its turn comes — never the one kept when its timer fired. Read before -// waiting, it gave back a module assigned a second earlier, and reported a declaration the mesh no -// longer recorded as sent. -func TestAReconcileAppliesWhatWasKeptWhenItsTurnComes(t *testing.T) { - built, err := system.For("arch") - if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { - t.Skip("this machine is not one these tests can apply on") - } - was := builtFor - builtFor = "arch" - t.Cleanup(func() { builtFor = was }) - dir := t.TempDir() - opts := options{state: filepath.Join(dir, "state.json")} - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - target := filepath.Join(dir, "a.conf") - kept := func(content string) store.Declared { - body := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + target + - `","content":"` + content + `\n"}]}`) - return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)} - } - if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("older")); err != nil { - t.Fatal(err) - } - - // The link is applying: the reconcile's timer fires and it waits its turn. - applying.Lock() - done := make(chan error, 1) - go func() { - _, err := reapplyKept(context.Background(), opts, public, nil, nil) - done <- err - }() - time.Sleep(50 * time.Millisecond) - // The link's apply ends by keeping the newer declaration, and lets go. - if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("newer")); err != nil { - applying.Unlock() - t.Fatal(err) - } - applying.Unlock() - - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(10 * time.Second): - t.Fatal("the reconcile never ran once the node was free") - } - got, err := os.ReadFile(target) - if err != nil { - t.Fatal(err) - } - if string(got) != "newer\n" { - t.Errorf("the reconcile applied %q, what was kept before the newer declaration", got) - } -} diff --git a/cmd/mesh-host/meshcli.go b/cmd/mesh-host/meshcli.go index c145a96..8960c27 100644 --- a/cmd/mesh-host/meshcli.go +++ b/cmd/mesh-host/meshcli.go @@ -11,14 +11,14 @@ import ( // serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account // from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served // is said once, and this engine goes on applying; mesh-cli then says the engine does not answer. -func serveMeshCLI(ctx context.Context, queue *link.Queue, say func(string)) { +func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) { if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil { say("mesh-cli is not answered on this machine: " + err.Error()) } } // forwardOver asks the controller over the queue's link, and follows the line to what came of it. -func forwardOver(queue *link.Queue) meshcli.Forward { +func forwardOver(queue *link.Session) meshcli.Forward { return func(ctx context.Context, asked meshcli.Asked) cli.Answer { return meshcli.AskTheController(ctx, queue.AskController, asked) } diff --git a/cmd/mesh-host/movesecret.go b/cmd/mesh-host/movesecret.go index 7bbc2ec..da6c23c 100644 --- a/cmd/mesh-host/movesecret.go +++ b/cmd/mesh-host/movesecret.go @@ -27,7 +27,7 @@ type vaultAsk func(verb string, args any) (json.RawMessage, error) // moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over // the link open now. -func moveSecretFrom(statePath string, queue *link.Queue) link.MoveSecretFunc { +func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc { return func(m link.MoveSecret) (link.MovedSecret, error) { key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath)) if err != nil { diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go new file mode 100644 index 0000000..71cc76f --- /dev/null +++ b/cmd/mesh-host/nodeagent.go @@ -0,0 +1,480 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup +// it asks and the apply of one module's composition. + +// bucketWatch is a bucket of the bus as the agent watches it, through the session's link. +type bucketWatch struct { + session *link.Session + bucket string +} + +func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Update, error) { + in, err := w.session.KVWatch(ctx, w.bucket, pattern) + if err != nil { + return nil, err + } + out := make(chan agent.Update, 64) + go func() { + defer close(out) + for u := range in { + select { + case out <- agent.Update{Key: u.Key, Value: u.Value, Revision: u.Revision, Deleted: u.Deleted, Caught: u.Caught}: + case <-ctx.Done(): + return + } + } + }() + return out, nil +} + +// sessionHaves writes the haves the agent says, through the session. +type sessionHaves struct{ session *link.Session } + +func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) { + got, ok := h.session.ReadHave(ctx, module) + if !ok { + return agent.Have{}, false + } + return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail}, true +} + +func (h sessionHaves) Write(ctx context.Context, x agent.Have) { + if x.Module == agent.NodeModule { + // The node's own key holds its facts and the digest of the composition it runs. + if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) { + h.session.NoteNode(ctx, x.Composed) + } + return + } + phase := x.Phase + if phase == string(delivery.PhaseRunning) { + phase = "" + } + h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase, + Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail)}) +} + +func (h sessionHaves) Removed(ctx context.Context, module string) { + h.session.NoteRemoved(ctx, module, "") +} + +// shorten keeps a have's detail within what a have value holds. +func shorten(s string) string { + const most = 300 + if len(s) <= most { + return s + } + return s[:most-1] + "…" +} + +// backupWithin is how long a backup is given from its ask to say it is done (ADR 0429 §14). +var backupWithin = 10 * time.Minute + +// backedUpEvent is where the holder of node-backup says a backup taken by hand ended, also one that could not start. +const backedUpEvent = "mesh.mod.restic.event.backed-up" + +// backupNode asks the holder of node-backup on this node to back a module up now, and waits for its event to say how it +// went: the holder answers `now` when it has started, and says `backed-up` when the backup ends. +func backupNode(session *link.Session) func(ctx context.Context, module string) agent.Backup { + return func(ctx context.Context, module string) agent.Backup { + // Heard before the ask, so an event that comes at once is not lost. + events, stop, err := session.Hear(backedUpEvent) + if err != nil { + return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()} + } + defer stop() + asked := time.Now() + if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil { + return agent.Backup{Words: err.Error()} + } + return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin) + } +} + +// awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it +// says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node +// or module, or from before the ask, is not the answer. +func awaitBackup(ctx context.Context, events <-chan []byte, node, module string, since time.Time, within time.Duration) agent.Backup { + bound := time.NewTimer(within) + defer bound.Stop() + for { + select { + case <-ctx.Done(): + return agent.Backup{Words: "the node agent was stopped while the backup ran"} + case <-bound.C: + return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)} + case raw := <-events: + var e struct { + Node string `json:"node"` + Module string `json:"module"` + OK bool `json:"ok"` + At time.Time `json:"at"` + Error string `json:"error"` + } + if json.Unmarshal(raw, &e) != nil || e.Node != node || e.Module != module || e.At.Before(since.Add(-time.Second)) { + continue + } + if e.OK { + return agent.Backup{Done: true} + } + words := "the backup failed" + if e.Error != "" { + words += ": " + e.Error + } + return agent.Backup{Words: words} + } + } +} + +// declarationOf is the declaration the compositions make together: their resources, and the node's own parts when the +// node's composition is among them. It is what the node judges and says its health by, and what one module's apply is +// made of (with a single composition). +func declarationOf(cs ...delivery.Composed) (*declaration.Declaration, error) { + d := &declaration.Declaration{Version: declaration.Version} + for _, c := range cs { + for _, raw := range c.Resources { + r, err := declaration.ParseResource(raw) + if err != nil { + return nil, fmt.Errorf("a resource of %q: %w", c.Module, err) + } + d.Resources = append(d.Resources, r) + } + d.Filled = append(d.Filled, c.Filled...) + if c.Module == "" { + d.Adoption = c.Own.Adoption + d.Identities = append(d.Identities, c.Own.Identities...) + d.Groups = append(d.Groups, c.Own.Groups...) + d.Libraries = append(d.Libraries, c.Own.Libraries...) + d.Provisioner = c.Own.Provisioner + } + } + return d, nil +} + +func scopeName(c delivery.Composed) string { + if c.Module == "" { + return agent.NodeModule + } + return c.Module +} + +func resourcesOf(cs []delivery.Composed) []declaration.Resource { + var out []declaration.Resource + for _, c := range cs { + for _, raw := range c.Resources { + if r, err := declaration.ParseResource(raw); err == nil { + out = append(out, r) + } + } + } + return out +} + +// operatorKeysFile is the file beside the node's state that holds the operator's public keys, one ed25519 key in base64 a +// line (a line that starts with # is a note). The switch's last declaration writes it as a file resource, beside the +// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption. +const operatorKeysFile = "operator-keys" + +// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the +// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at +// the next composition. A file that cannot be read trusts the controller's alone, and says so. +func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey { + return func() []ed25519.PublicKey { + keys := []ed25519.PublicKey{} + if len(controller) == ed25519.PublicKeySize { + keys = append(keys, controller) + } + raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile)) + if err != nil { + return keys + } + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, err := base64.StdEncoding.DecodeString(line) + if err != nil || len(key) != ed25519.PublicKeySize { + fmt.Fprintf(os.Stderr, "mesh-host: a line of %s is not an ed25519 public key in base64, and is not trusted\n", operatorKeysFile) + continue + } + keys = append(keys, ed25519.PublicKey(key)) + } + return keys + } +} + +// keptByModule is the resources of the declaration the node kept before the switch, by the module they say; the node's +// own are those that say none (ADR 0539 §9). +func keptByModule(opts options, signer ed25519.PublicKey) func() map[string][]json.RawMessage { + var once sync.Once + var by map[string][]json.RawMessage + return func() map[string][]json.RawMessage { + once.Do(func() { + raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) + if err != nil { + return + } + var kept struct { + Resources []json.RawMessage `json:"resources"` + } + if json.Unmarshal(raw, &kept) != nil { + return + } + by = map[string][]json.RawMessage{} + for _, r := range kept.Resources { + var head struct { + Module string `json:"module"` + } + _ = json.Unmarshal(r, &head) + name := head.Module + if name == "" { + name = agent.NodeModule + } + by[name] = append(by[name], r) + } + }) + return by + } +} + +// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the +// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's +// account may not read) is said, and tried again with a growing wait, as the link is. +func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler, + say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) { + wait := 2 * time.Second + for ctx.Err() == nil { + linked, cancel, err := session.WhenLinked(ctx) + if err != nil { + return + } + began := time.Now() + node := session.Membership.Node + composedBucket, err := delivery.ComposedBucketFor(node) + if err != nil { + cancel() + say("the node agent cannot name this node's composition bucket: " + err.Error()) + return + } + var ag *agent.Agent + env := agent.Env{ + Node: node, + Wants: bucketWatch{session, delivery.AssignerWantsBucket}, + Composed: bucketWatch{session, composedBucket}, + Haves: sessionHaves{session}, + Kept: keptByModule(opts, signer), + Trusted: trustedKeys(opts, signer), + Backup: backupNode(session), + Say: say, + Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result { + res := applyComposition(session.WithLink(ctx), opts, c, others, sched, say) + afterApply() + sayFacts(ctx, ag) + return res + }, + Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result { + res := removeComposition(session.WithLink(ctx), opts, module, others, sched, say) + afterApply() + sayFacts(ctx, ag) + return res + }, + } + ag = agent.New(env) + current.set(ag) + err = ag.Run(linked) + cancel() + current.set(nil) + if ctx.Err() != nil { + return + } + if time.Since(began) > 30*time.Second { + wait = 2 * time.Second + } + if err != nil { + say(fmt.Sprintf("the node agent stopped: %v — beginning again in %s", err, wait)) + } + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + if wait *= 2; wait > 2*time.Minute { + wait = 2 * time.Minute + } + } +} + +// agentNow is the agent running now, which the node's facts are built from. +type agentNow struct { + mu sync.Mutex + ag *agent.Agent +} + +func (c *agentNow) set(a *agent.Agent) { + c.mu.Lock() + c.ag = a + c.mu.Unlock() +} + +func (c *agentNow) get() *agent.Agent { + c.mu.Lock() + defer c.mu.Unlock() + return c.ag +} + +// applyComposition applies one composition: a module's, or the node's own. +func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, + sched *apply.Scheduler, say link.Announce) agent.Result { + name := scopeName(c) + d, err := declarationOf(c) + if err != nil { + return agent.Result{Failed: true, Detail: shorten(err.Error())} + } + scope := apply.Scope{Module: name, Others: resourcesOf(others)} + got := applyModuleHeld(ctx, opts, d, &scope, sched, say) + if got.Refused != "" { + return agent.Result{Failed: true, Detail: shorten(got.Refused)} + } + results := apply.ModuleResults(d, got.Outcome, got.Err) + if r, ok := results[name]; ok && r.Failed { + return agent.Result{Failed: true, Detail: shorten(r.Detail)} + } + if got.Err != nil { + return agent.Result{Failed: true, Detail: shorten(got.Err.Error())} + } + if got.Probe != nil { + return agent.Result{Failed: true, Detail: shorten(got.Probe.partial().Error())} + } + return agent.Result{} +} + +// removeComposition removes one module's own resources and nothing of another's. +func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, + sched *apply.Scheduler, say link.Announce) agent.Result { + scope := apply.Scope{Module: module, Others: resourcesOf(others)} + got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say) + switch { + case got.Refused != "": + return agent.Result{Failed: true, Detail: shorten(got.Refused)} + case got.Err != nil: + return agent.Result{Failed: true, Detail: shorten(got.Err.Error())} + } + return agent.Result{} +} + +// busyModules are the modules whose last apply left a container running because it said busy. +func busyModules() []string { + busyMu.Lock() + defer busyMu.Unlock() + var out []string + for m, w := range busyBy { + if w && m != "" { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given. +func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + var raw []byte + if len(signer) > 0 { + var err error + if raw, err = store.LoadDeclared(store.DeclaredPath(statePath), signer); err != nil { + return nil, err + } + } else { + kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + if err != nil { + return nil, err + } + raw = kept.Declaration + } + return declaration.Parse(raw) +} + +// whatRuns is what runs on this node as the declaration of it (novox/hq ADR 0539 §9): the compositions the agent runs +// and, for every module the agent does not hold yet, what the kept declaration has for it, with the node's own parts +// (adoption, identities, groups, libraries) from the node's composition once the agent holds it and from the kept +// declaration until then. What the node judges its health by, and what the filling, the take act and the judgement of +// directories read. +func whatRuns(everything []delivery.Composed, holdsNode bool, kept func() (*declaration.Declaration, error)) (*declaration.Declaration, error) { + d, err := declarationOf(everything...) + if err != nil { + return nil, err + } + if holdsNode { + return d, nil + } + old, err := kept() + if err != nil { + if len(everything) > 0 { + return d, nil + } + return nil, err + } + d.Adoption, d.Identities, d.Groups, d.Libraries = old.Adoption, old.Identities, old.Groups, old.Libraries + d.Provisioner, d.Builds = old.Provisioner, old.Builds + d.Filled = append(d.Filled, old.Filled...) + return d, nil +} + +// runsHere says what runs on this node; the running serve sets it to the agent's view, and a command run by hand reads +// the kept declaration. +var runsHere struct { + mu sync.Mutex + fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) +} + +func setRunsHere(fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error)) { + runsHere.mu.Lock() + runsHere.fn = fn + runsHere.mu.Unlock() +} + +// declaredNow is the declaration of what runs on this node now. +func declaredNow(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + runsHere.mu.Lock() + fn := runsHere.fn + runsHere.mu.Unlock() + if fn != nil { + return fn(statePath, signer) + } + return keptDeclaration(statePath, signer) +} + +// runsFromAgent is what runs now as the agent in hand sees it. +func runsFromAgent(current *agentNow) func(string, ed25519.PublicKey) (*declaration.Declaration, error) { + return func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + kept := func() (*declaration.Declaration, error) { return keptDeclaration(statePath, signer) } + ag := current.get() + if ag == nil { + return kept() + } + return whatRuns(ag.Everything(), ag.HoldsNode(), kept) + } +} diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go new file mode 100644 index 0000000..f53efb3 --- /dev/null +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -0,0 +1,313 @@ +//go:build integration + +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/link" +) + +// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition +// bucket and the have bucket, a session on the bus, and the real apply of a module's composition into a directory of the +// test. R6-A AC1, AC5 and AC9 (novox/hq ADR 0539). + +const itNode = "it1" + +type itBed struct { + t *testing.T + conn *nats.Conn + js nats.JetStreamContext + wants nats.KeyValue + composed nats.KeyValue + haves nats.KeyValue + dir string + opts options + session *link.Session + controll ed25519.PrivateKey + operator ed25519.PrivateKey + said []string + mu sync.Mutex +} + +func aServer(t *testing.T) *nats.Conn { + t.Helper() + srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the embedded server did not start") + } + t.Cleanup(srv.Shutdown) + conn, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + return conn +} + +func newItBed(t *testing.T) *itBed { + t.Helper() + // Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node. + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + was := builtFor + builtFor = "arch" + t.Cleanup(func() { builtFor = was }) + + b := &itBed{t: t, conn: aServer(t), dir: t.TempDir()} + var err error + if b.js, err = b.conn.JetStream(); err != nil { + t.Fatal(err) + } + composedBucket, _ := delivery.ComposedBucketFor(itNode) + for _, c := range []struct { + name string + into *nats.KeyValue + history uint8 + }{{delivery.AssignerWantsBucket, &b.wants, 8}, {composedBucket, &b.composed, delivery.ComposedHistory}, + {delivery.HaveBucket, &b.haves, delivery.HaveHistory}} { + kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: c.name, History: c.history}) + if err != nil { + t.Fatal(err) + } + *c.into = kv + } + b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second} + if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil { + t.Fatal(err) + } + _, b.controll, _ = ed25519.GenerateKey(nil) + _, b.operator, _ = ed25519.GenerateKey(nil) + return b +} + +// start runs the node agent under a session on the bus, trusting the controller's key, and the operator's when asked. +func (b *itBed) start(trustOperator bool) (stop func()) { + b.t.Helper() + if trustOperator { + pub := b.operator.Public().(ed25519.PublicKey) + if err := os.WriteFile(filepath.Join(filepath.Dir(b.opts.state), operatorKeysFile), + []byte("# the operator\n"+base64.StdEncoding.EncodeToString(pub)+"\n"), 0o600); err != nil { + b.t.Fatal(err) + } + } + b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, + Say: b.say} + b.session.Attach(context.Background(), link.OverNATS{Conn: b.conn, JS: b.js, Node: itNode}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {}, + &agentNow{}, func(context.Context, *agent.Agent) {}) + }() + return func() { cancel(); <-done } +} + +func (b *itBed) say(line string) { + b.mu.Lock() + b.said = append(b.said, line) + b.mu.Unlock() + b.t.Log(line) +} + +func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } + +// compose writes a module's signed composition of one file into the node's bucket and returns its digest. +func (b *itBed) compose(module, fp, content string, data bool, signer ed25519.PrivateKey) string { + b.t.Helper() + path := filepath.Join(b.dir, module+".conf") + res, _ := json.Marshal(map[string]any{"id": module + ".conf", "type": "file", "path": path, "content": content, "module": module}) + c := delivery.Composed{Node: itNode, Module: module, FP: fp, Data: data, Resources: []json.RawMessage{res}} + signed, err := delivery.SignComposed(c, signer) + if err != nil { + b.t.Fatal(err) + } + raw, err := delivery.EncodeComposed(signed) + if err != nil { + b.t.Fatal(err) + } + dig, _ := delivery.ComposedDigestOf(signed) + key, _ := delivery.ComposedKey(itNode, module, fp) + if _, err := b.composed.Put(key, raw); err != nil { + b.t.Fatal(err) + } + return dig +} + +func (b *itBed) want(module, fp, digest, backup string) { + b.t.Helper() + raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup}) + if err != nil { + b.t.Fatal(err) + } + key, _ := delivery.WantKey(itNode, module) + if _, err := b.wants.Put(key, raw); err != nil { + b.t.Fatal(err) + } +} + +func (b *itBed) file(module string) string { + got, _ := os.ReadFile(filepath.Join(b.dir, module+".conf")) + return string(got) +} + +func (b *itBed) have(module string) delivery.Have { + key, _ := delivery.HaveKey(itNode, module) + e, err := b.haves.Get(key) + if err != nil { + return delivery.Have{} + } + h, _ := delivery.DecodeHave(e.Value()) + return h +} + +func within(t *testing.T, d time.Duration, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(d) + for !ok() { + if time.Now().After(deadline) { + t.Fatal(what) + } + time.Sleep(10 * time.Millisecond) + } +} + +// R6-A AC1: given a running agent on a bus, when a want naming a digest and that composition are written, the module is +// applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names +// is not applied. +func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) { + b := newItBed(t) + stop := b.start(false) + defer stop() + time.Sleep(300 * time.Millisecond) // the watches are up + + fp := fpIt("a") + dig := b.compose("web", fp, "one\n", false, b.controll) + time.Sleep(300 * time.Millisecond) + if b.file("web") != "" { + t.Fatal("a composition no want names was applied") + } + began := time.Now() + b.want("web", fp, dig, "") + within(t, time.Second, "the module was not applied within one second of its want", func() bool { return b.file("web") == "one\n" }) + t.Logf("applied %s after the want", time.Since(began)) + within(t, 2*time.Second, "the have does not name the fingerprint and the digest", func() bool { + h := b.have("web") + return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning + }) + + // A want naming a digest the composition at its fingerprint does not have applies nothing. + fp2 := fpIt("b") + b.compose("web", fp2, "two\n", false, b.controll) + b.want("web", fp2, "sha256:"+strings.Repeat("0", 64), "") + time.Sleep(300 * time.Millisecond) + if b.file("web") != "one\n" { + t.Fatalf("a want naming another digest changed the module: %q", b.file("web")) + } +} + +// R6-A AC5: a data module that ran before asks its backup before any resource changes; a backup that fails holds the +// module (nothing applied, the have says held with the backup's words); when its want then says skip, it is applied. +func TestNatsADataModuleAsksItsBackupFirstAndIsHeldWhenItFails(t *testing.T) { + b := newItBed(t) + var mu sync.Mutex + var asked []string + var onDiskWhenAsked string + if _, err := b.conn.Subscribe("mesh.seat.node-backup.tool.now."+itNode, func(m *nats.Msg) { + mu.Lock() + asked = append(asked, "now") + onDiskWhenAsked = b.file("db") + mu.Unlock() + _ = m.Respond([]byte(`{"result":{"started":"db"}}`)) + // The holder says the backup ended on its event. + ev, _ := json.Marshal(map[string]any{"node": itNode, "module": "db", "ok": false, "at": time.Now().UTC(), + "error": "restic: the repository is locked"}) + _ = b.conn.Publish(backedUpEvent, ev) + }); err != nil { + t.Fatal(err) + } + stop := b.start(false) + defer stop() + time.Sleep(300 * time.Millisecond) + + fp1, fp2 := fpIt("a"), fpIt("b") + b.want("db", fp1, b.compose("db", fp1, "v1\n", true, b.controll), "") + within(t, 3*time.Second, "the first composition was not applied", func() bool { return b.file("db") == "v1\n" }) + mu.Lock() + if len(asked) != 0 { + t.Fatalf("a module that never ran was backed up: %v", asked) + } + mu.Unlock() + + dig2 := b.compose("db", fp2, "v2\n", true, b.controll) + b.want("db", fp2, dig2, "") + within(t, 3*time.Second, "the module was not held behind the failed backup", func() bool { + h := b.have("db") + return h.Phase == delivery.PhaseHeld && strings.Contains(h.Detail, "repository is locked") + }) + mu.Lock() + if len(asked) != 1 || onDiskWhenAsked != "v1\n" { + t.Fatalf("the backup was asked %v, with %q on disk: it is asked before any resource changes", asked, onDiskWhenAsked) + } + mu.Unlock() + if b.file("db") != "v1\n" { + t.Fatalf("something was applied behind a failed backup: %q", b.file("db")) + } + if h := b.have("db"); h.FP != fp1 { + t.Fatalf("a held module should still name what it runs: %+v", h) + } + + b.want("db", fp2, dig2, delivery.BackupSkip) + within(t, 3*time.Second, "the module was not applied once its want said skip", func() bool { return b.file("db") == "v2\n" }) + within(t, 2*time.Second, "the have does not say running at the new fingerprint", func() bool { + h := b.have("db") + return h.Phase == delivery.PhaseRunning && h.FP == fp2 && h.Composed == dig2 + }) +} + +// R6-A AC9, the hand path end to end: with no controller running, a composition signed by the operator's key and a want +// naming it are applied and the have written; the same signed by a key nobody trusts is not. +func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) { + b := newItBed(t) + stop := b.start(true) + defer stop() + time.Sleep(300 * time.Millisecond) + + _, stranger, _ := ed25519.GenerateKey(nil) + fpS := fpIt("c") + b.want("evil", fpS, b.compose("evil", fpS, "no\n", false, stranger), "") + time.Sleep(300 * time.Millisecond) + if b.file("evil") != "" { + t.Fatal("a composition signed by a key nobody trusts was applied") + } + + fp := fpIt("d") + dig := b.compose("rescue", fp, "back\n", false, b.operator) + b.want("rescue", fp, dig, "") + within(t, 2*time.Second, "the operator's composition was not applied", func() bool { return b.file("rescue") == "back\n" }) + within(t, 2*time.Second, "the have was not written", func() bool { + h := b.have("rescue") + return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning + }) +} diff --git a/cmd/mesh-host/nodeclare_test.go b/cmd/mesh-host/nodeclare_test.go new file mode 100644 index 0000000..430fec3 --- /dev/null +++ b/cmd/mesh-host/nodeclare_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// R6-A AC8: nothing in the node-engine receives, orders or reconciles a whole declaration (novox/hq ADR 0539 §9, ADR +// 0429 §8): no package subscribes to a node's declare subject or names the deleted symbols. +func TestNoPackageHearsOrOrdersAWholeDeclaration(t *testing.T) { + deleted := []string{ + "DeclareSubject", "mesh.node.\" + node + \".declare", "ReconcileDue", "ReconcileEvery", "olderThanKept", + "refuseOlder", "olderGeneration", "appliedGeneration", "reconcileKept", "reapplyKept", "adoptionWatch", + "unsaidFile", "numbersFile", "func gather(", "func handleBody(", "Supersedes(", "ReportSequence", + "RefusedOlder", "ReadsGeneration", "ReadsModules", "GenerationRefusal", "LeftOutModuleOf", + } + root := filepath.Join("..", "..") + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for _, word := range deleted { + if strings.Contains(string(body), word) { + t.Errorf("%s still names %q", path, word) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/mesh-host/order_test.go b/cmd/mesh-host/order_test.go deleted file mode 100644 index 1cf9d90..0000000 --- a/cmd/mesh-host/order_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq -// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said, -// "older" could not. - -func keptWith(t *testing.T, sequence int64) store.Declared { - t.Helper() - return keptAt(t, link.Order{Sequence: sequence}) -} - -func keptAt(t *testing.T, order link.Order) store.Declared { - t.Helper() - body, err := json.Marshal(map[string]any{ - "declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence, - "epoch": order.Epoch, - }) - if err != nil { - t.Fatal(err) - } - return store.Declared{Declaration: body, Signature: []byte("x")} -} - -func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) { - err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5}) - if err == nil { - t.Fatal("sequence 5 was accepted over a kept 7") - } - if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") { - t.Fatalf("the refusal does not say what it is: %v", err) - } -} - -func TestANewerOrEqualDeclarationIsNot(t *testing.T) { - if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil { - t.Fatalf("sequence 8 was refused over a kept 7: %v", err) - } - // Equal is the same declaration again, which reconciling is for. - if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil { - t.Fatalf("the same sequence was refused: %v", err) - } -} - -func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) { - // An older controller sends none; a host that received before it understood them kept none. - // Refusing on a guess would strand a node the moment the controller is older than the host. - if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil { - t.Fatalf("a declaration claiming no order was refused: %v", err) - } - if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil { - t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err) - } - if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil { - t.Fatalf("a first declaration was refused: %v", err) - } -} - -// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost -// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence. -func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) { - kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3}) - err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12}) - if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") { - t.Fatalf("an older epoch was not refused by name: %v", err) - } - if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil { - t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err) - } - if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil { - t.Fatal("a lower sequence under the same epoch was accepted") - } - // A controller with no lease — older, or rolled back to a build from before it — is today's - // behaviour: no epoch compared. - if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil { - t.Fatalf("a declaration with no epoch was refused for one: %v", err) - } - if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older || - held != (link.Order{Epoch: 57, Sequence: 3}) { - t.Fatalf("olderThanKept said %v, holding %+v", older, held) - } - // What is kept cannot be read: no order to compare, so nothing is refused on a guess. - if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil, - link.Order{Epoch: 41, Sequence: 1}); older { - t.Fatal("refused against a kept declaration nobody can read") - } -} diff --git a/cmd/mesh-host/probe_test.go b/cmd/mesh-host/probe_test.go index 6433fd2..357aa1e 100644 --- a/cmd/mesh-host/probe_test.go +++ b/cmd/mesh-host/probe_test.go @@ -9,7 +9,6 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/declaration" - "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/system" ) @@ -127,83 +126,3 @@ func TestAPackageManagerThatIsNotThereRefusesWhole(t *testing.T) { t.Error("the refused declaration touched the machine") } } - -// Defends the review of novox/hq issue 233: **a re-apply whose result changed is said unasked**. A machine -// reconciles every five minutes and speaks only when its account is news; how the apply went was not part of -// it, so a package manager that stopped answering between deliveries was never said, and one that answered -// again never cleared. Now a failure is said once, the same failure again is not, and a clean re-apply after it -// is — which is what clears the mesh's condition. -func TestAReapplyWhoseResultChangedIsSaidUnasked(t *testing.T) { - dir := t.TempDir() - conf := filepath.Join(dir, "elsewhere", "pacman.conf") // not declared: the apply cannot make it answer - pacmanAnsweringOnce(t, conf) - opts := options{state: filepath.Join(dir, "state.json")} - raw := []byte(`{"declaration":1,"resources":[` + - `{"id":"motd","type":"file","path":"` + filepath.Join(dir, "motd") + `","content":"hello\n"},` + - `{"id":"pacman.base","type":"package","package":"base"}]}`) - w := &adoptionWatch{} - - // Clean once, as it was before: base installed while pacman answered. - if err := os.MkdirAll(filepath.Dir(conf), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil { - t.Fatal(err) - } - clean := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) - if faulty(clean) { - t.Fatalf("the first apply was not clean: %+v", clean) - } - w.said(clean) - if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) { - t.Error("a clean re-apply after a clean one was said, so every machine speaks every five minutes") - } - - // The configuration goes: the re-apply fails, and that is news once. - if err := os.Remove(conf); err != nil { - t.Fatal(err) - } - failed := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) - if !faulty(failed) { - t.Fatalf("the re-apply did not fail: %+v", failed) - } - if !w.news(failed) { - t.Fatal("a re-apply that failed after a clean one was not said, so the mesh never hears the machine fails") - } - w.said(failed) - if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) { - t.Error("the same failure was said again on the next re-apply") - } - - // It comes back: the clean re-apply is news, and is what clears the mesh's condition. - if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil { - t.Fatal(err) - } - again := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) - if faulty(again) { - t.Fatalf("the re-apply did not come back clean: %+v", again) - } - if !w.news(again) { - t.Fatal("a clean re-apply after a failed one was not said, so the mesh's condition never clears") - } -} - -// A refusal is news too, once, and a reconcile with nothing to say after a clean one stays silent. -func TestARefusalIsSaidOnceAndNothingIsNotSaid(t *testing.T) { - w := &adoptionWatch{} - w.said(link.Report{}) - if w.news(link.Report{}) { - t.Error("nothing was said again") - } - refused := link.Report{Refused: "pacman does not answer here"} - if !w.news(refused) { - t.Fatal("a refusal was not said") - } - w.said(refused) - if w.news(link.Report{Refused: "pacman does not answer here, at another time"}) { - t.Error("the same refusal in other words was said again") - } - if !w.news(link.Report{}) { - t.Error("a clean apply after a refusal was not said") - } -} diff --git a/cmd/mesh-host/unsaid_test.go b/cmd/mesh-host/unsaid_test.go deleted file mode 100644 index 4c63317..0000000 --- a/cmd/mesh-host/unsaid_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" -) - -// The last apply's report, kept beside the state until the mesh takes it (novox/hq issue 264). - -// What a host that died between applying and reporting kept is read back by the next one exactly — -// the digest and the outcome, never a new account of the machine. -func TestTheKeptReportSurvivesTheHost(t *testing.T) { - state := filepath.Join(t.TempDir(), "state.json") - made := link.Report{Declared: "sha256:d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"}, - Host: "v2"} - if err := unsaidBeside(state).Keep(made); err != nil { - t.Fatal(err) - } - - kept, ok, err := unsaidBeside(state).Pending() - if err != nil || !ok { - t.Fatalf("nothing kept: %v", err) - } - if !reflect.DeepEqual(kept, made) { - t.Fatalf("kept %+v, made %+v", kept, made) - } - - // Taken about another declaration, it stays; taken about this one, it goes. - if err := unsaidBeside(state).Said("sha256:other"); err != nil { - t.Fatal(err) - } - if _, ok, _ := unsaidBeside(state).Pending(); !ok { - t.Fatal("a report about another declaration cleared this one") - } - if err := unsaidBeside(state).Said("sha256:d1"); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(store.UnsaidPath(state)); !os.IsNotExist(err) { - t.Fatalf("the report was taken and is still kept: %v", err) - } -} - -// A node that never applied anything has nothing to say again; and an apply refused before its -// declaration was read clears an older report, which would otherwise be re-said after it. -func TestNothingKeptWhenNothingWasApplied(t *testing.T) { - state := filepath.Join(t.TempDir(), "state.json") - if _, ok, err := unsaidBeside(state).Pending(); ok || err != nil { - t.Fatalf("a node that applied nothing has a report to say: %v %v", ok, err) - } - if err := unsaidBeside(state).Keep(link.Report{Declared: "sha256:d1"}); err != nil { - t.Fatal(err) - } - if err := unsaidBeside(state).Keep(link.Report{Refused: "forged"}); err != nil { - t.Fatal(err) - } - if _, ok, _ := unsaidBeside(state).Pending(); ok { - t.Fatal("an older report is kept after a later declaration was refused") - } -} diff --git a/cmd/mesh-host/vaultbox_nats_test.go b/cmd/mesh-host/vaultbox_nats_test.go index 719040c..30e0fd5 100644 --- a/cmd/mesh-host/vaultbox_nats_test.go +++ b/cmd/mesh-host/vaultbox_nats_test.go @@ -22,7 +22,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/identity" - "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) @@ -61,14 +60,14 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear( } _ = tap.Flush() - // The engine hears its declaration on its own subject. - const node = "vaultbox" + // The engine hears the composition on a subject of the test (the node agent reads it from its bucket). + const subject = "mesh.test.vaultbox.composition" engine, err := nats.Connect(url) if err != nil { t.Fatal(err) } defer engine.Close() - heard, err := engine.SubscribeSync(link.DeclareSubject(node)) + heard, err := engine.SubscribeSync(subject) if err != nil { t.Fatal(err) } @@ -92,7 +91,7 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear( t.Fatal(err) } defer controller.Close() - if err := controller.Publish(link.DeclareSubject(node), body); err != nil { + if err := controller.Publish(subject, body); err != nil { t.Fatal(err) } _ = controller.Flush() @@ -123,7 +122,7 @@ func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear( mu.Lock() all := strings.Join(crossed, "\n") mu.Unlock() - if !strings.Contains(all, link.DeclareSubject(node)) || !strings.Contains(all, "vault-sealed") { + if !strings.Contains(all, subject) || !strings.Contains(all, "vault-sealed") { t.Fatalf("the listener heard nothing of the declaration:\n%s", all) } for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} { diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go new file mode 100644 index 0000000..2b49c0e --- /dev/null +++ b/cmd/mesh-host/whatruns_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func aFileOf(module, id string) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id, + "content": "x", "module": module}) + return raw +} + +// What runs is the agent's compositions and the kept declaration's resources for each module the agent does not hold; +// the node's own parts are the kept declaration's until the agent holds the node's composition (ADR 0539 §9). One module +// under the agent must not make the rest invisible to the liveness, units, accounts, libraries, filling and the take act. +func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) { + kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}}, + Libraries: []declaration.Library{{Library: "media"}}} + everything := []delivery.Composed{ + {Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}}, + {Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's + } + d, err := whatRuns(everything, false, func() (*declaration.Declaration, error) { return kept, nil }) + if err != nil { + t.Fatal(err) + } + if len(d.Resources) != 2 || len(d.Identities) != 1 || len(d.Libraries) != 1 { + t.Fatalf("%d resources, %d identities, %d libraries: the kept modules and the node's kept parts are missing", + len(d.Resources), len(d.Identities), len(d.Libraries)) + } + + // Once the agent holds the node's composition, its parts are the node's. + node := delivery.Composed{Node: "n1", Own: delivery.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}} + d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") }) + if err != nil { + t.Fatal(err) + } + if len(d.Identities) != 1 || d.Identities[0].Name != "mesh-web" || len(d.Libraries) != 0 { + t.Fatalf("the node's parts: %+v %+v", d.Identities, d.Libraries) + } +} + +// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration. +func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { + setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) { + return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil + }) + defer setRunsHere(nil) + d, err := declaredNow("/nowhere", nil) + if err != nil || d.Provisioner != "from-agent" { + t.Fatalf("declaredNow read %+v (%v), not what runs", d, err) + } +} diff --git a/cmd/mesh-host/witnessing.go b/cmd/mesh-host/witnessing.go index bf7bffb..d4aa7a0 100644 --- a/cmd/mesh-host/witnessing.go +++ b/cmd/mesh-host/witnessing.go @@ -41,8 +41,9 @@ func rolledBackHosts(statePath string) []string { return upgrade.RolledBackVersions(verdicts) } -// proving is this host waiting for the mesh to take a report it made under its own build — the -// evidence its launcher's trial waits for — and acting on it once. +// proving is this host waiting to write a have key under its own build — the evidence its launcher's trial waits for +// (novox/hq ADR 0539 §9, ADR 0227 §8: the component replaced is never the only witness of its successor) — and acting +// on it once. type proving struct { statePath string version string @@ -56,16 +57,14 @@ func unproved(statePath, version string) bool { return err != nil || known != version } -// heard is told every account of the machine the mesh has taken. The first one about a declaration, -// made by this build, proves it. -func (p *proving) heard(r link.Report) { - if r.Declared == "" || r.Refused != "" || r.Host != p.version { - return - } +// haveWritten is told when this build has written a have key, or found the bucket already holding what it would write. +// The first proves it. **A declaration's report does not**: a node-engine that receives none is not on trial for it, and +// one that cannot write its haves is what the launcher keeps rolling back. +func (p *proving) haveWritten() { p.once.Do(func() { retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version) if err != nil { - p.say(fmt.Sprintf("this host reported under its own build %s, and proving it was not complete: %v", p.version, err)) + p.say(fmt.Sprintf("this host wrote its first have under its own build %s, and proving it was not complete: %v", p.version, err)) } if len(retired) > 0 { p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", "))) @@ -146,7 +145,7 @@ func launcherReplaced(pid int, proc, path string) bool { } // watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8). -func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue, say func(string)) { +func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) { host, _ := os.Hostname() w := &witness.Watcher{ Root: apply.DaemonRoot(), Node: node, Host: host, @@ -159,7 +158,7 @@ func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue }, // Said now, not at the next report: the verdict is on every report from here, and the mesh // is asked for one at once. - Concluded: func(link.Rollback) { queue.ReportAsked() }, + Concluded: func(link.Rollback) { queue.FactsAsked(ctx) }, Say: say, } w.Watch(ctx) diff --git a/cmd/mesh-host/witnessing_test.go b/cmd/mesh-host/witnessing_test.go index 13ee112..3e304b6 100644 --- a/cmd/mesh-host/witnessing_test.go +++ b/cmd/mesh-host/witnessing_test.go @@ -13,9 +13,9 @@ import ( "github.com/novox/mesh-host/internal/upgrade" ) -// The first account the mesh takes from this build, about a declaration, proves it to the launcher; -// nothing else does — a refusal, a report about no declaration, another build's report (to-be 45 §8). -func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) { +// The first have this build writes proves it to the launcher; nothing else does — not a report about a declaration, not +// a facts event, not the haves of another build (novox/hq ADR 0539 §9, ADR 0227 §8). R6-A AC7. +func TestOnlyAHaveUnderThisBuildProvesIt(t *testing.T) { state := filepath.Join(t.TempDir(), "state.json") t.Setenv("MESH_HOST_LIBEXEC", t.TempDir()) // Never the machine's own engine placed by hand: proving renews it (issue 356). @@ -27,22 +27,20 @@ func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) { var said []string p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }} - for _, r := range []link.Report{ - {Host: "2.0", Refused: "no"}, - {Host: "2.0"}, - {Host: "1.0", Declared: "abc"}, - } { - p.heard(r) - if !unproved(state, "2.0") { - t.Fatalf("%+v proved this build", r) - } - if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" { - t.Fatalf("%+v renewed the engine placed by hand", r) - } + // Facts said, a link opened, a declaration of the old kind reported: none of it is a have. + if !unproved(state, "2.0") { + t.Fatal("a build that wrote no have is proved") } - p.heard(link.Report{Host: "2.0", Declared: "abc"}) + if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" { + t.Fatal("the engine placed by hand was renewed before any have") + } + + // The session tells the proof when it has a have key saying what it would write. + session := &link.Session{HaveWritten: p.haveWritten} + _ = session + p.haveWritten() if unproved(state, "2.0") { - t.Fatalf("an account of a declaration under this build did not prove it (%v)", said) + t.Fatalf("the first have under this build did not prove it (%v)", said) } if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" { t.Fatalf("known-good is %q", got) @@ -78,12 +76,6 @@ func TestAStandingRollbackIsSaidOnEveryReport(t *testing.T) { got[0].Outcome != link.RolledBack { t.Fatalf("what the report says is %+v", got) } - if !worthSaying(link.Report{Rollbacks: got}) { - t.Fatal("a reconcile with a rollback standing does not say it") - } - if adoptionFingerprint(link.Report{Rollbacks: got}) == adoptionFingerprint(link.Report{}) { - t.Fatal("a rollback reached or ended is not news to the reconcile") - } } // The launcher running this host is known to have been replaced only when it certainly was: a shell diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 142231b..5c4bc62 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -221,7 +221,7 @@ "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0600", "create-once": true, - "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" + "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the // mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned diff --git a/go.mod b/go.mod index df20074..65c0bb2 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,20 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c + github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 ) require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index 6ecfe04..d36d16c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,17 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75 h1:AcYfyJpYaUufhI93VG76+b001msXY4XX7kuRjnf33HY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= @@ -10,5 +20,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/internal/agent/agent.go b/internal/agent/agent.go new file mode 100644 index 0000000..8c472d9 --- /dev/null +++ b/internal/agent/agent.go @@ -0,0 +1,619 @@ +// Package agent is the node agent (novox/hq ADR 0429 §4, ADR 0539): the node-engine's own process that follows the +// node's wants and applies one module at a time. +// +// **It watches, it never polls.** A want `want..` names a fingerprint and the digest of the one composition +// the assigner allows the node to apply for it; the composition itself waits under `..` in +// the node's bucket `composed-`. When both are there and the composition's digest is the want's, and what runs +// differs, the agent applies that module's composition and writes its have with the fingerprint and the digest. A want +// `removed` removes that module's own resources. Nothing is applied for a composition no want names, and a failed apply +// is retried by the next change of either key, never by a clock (research 063 §9.1). +package agent + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "sort" + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Update is one change of a key in a watched bucket. Caught marks the end of the keys that were there when the watch +// began; every Update after it is a change. +type Update struct { + Key string + Value []byte + Revision uint64 + Deleted bool + Caught bool +} + +// Watcher follows one bucket: the keys matching the pattern now, a Caught update, then every change, until ctx ends. +type Watcher interface { + Watch(ctx context.Context, pattern string) (<-chan Update, error) +} + +// NodeModule is the scope of the node's own composition, `.node`. +const NodeModule = "node" + +// Have is what a module's have key says, as the agent reads and writes it. +type Have struct { + Module string + FP string + Composed string + // Phase is "running", "failed", "held" or "removed"; empty reads as running. + Phase string + Detail string +} + +// Haves is the node's have bucket as the agent uses it. +type Haves interface { + // Read is what the node-engine last wrote for a module, and whether it wrote anything. + Read(ctx context.Context, module string) (Have, bool) + // Write writes a module's have where it differs from the bucket's. + Write(ctx context.Context, h Have) + // Removed writes the module's have with the phase removed, once, if a key of it exists. + Removed(ctx context.Context, module string) +} + +// Result is how one apply or removal came out. +type Result struct { + Failed bool + Detail string +} + +// Backup is the answer of the backup asked before a module moves. +type Backup struct { + Done bool + Words string +} + +// Env is everything the agent needs of the node. +type Env struct { + Node string + Wants Watcher + Composed Watcher + Haves Haves + // Apply applies one module's composition (scope NodeModule for the node's own), given the compositions the node's + // other modules run so a path, unit or group they hold is not removed or taken. It is called with one module at a + // time. + Apply func(ctx context.Context, c delivery.Composed, others []delivery.Composed) Result + // Remove removes the module's own resources and nothing of another's. + Remove func(ctx context.Context, module string, others []delivery.Composed) Result + // Backup backs a module up before a new composition of it is applied; nil backs up nothing. + Backup func(ctx context.Context, module string) Backup + // Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539 + // §9), as JSON; the node's own are under NodeModule. Nil when the node kept none. A module the agent does not hold yet + // still runs from it. + Kept func() map[string][]json.RawMessage + // Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A + // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is + // applied. Nil trusts no key, so nothing is applied. + Trusted func() []ed25519.PublicKey + Say func(string) +} + +// Agent follows the wants. +type Agent struct { + env Env + + mu sync.Mutex + wants map[string]wantNow // by module + comps map[string]compNow // by composed key + running map[string]delivery.Have + // attempt is the last apply of a module that failed or is held, with the revisions it was made from. + attempt map[string]attempt + checked map[string]bool + // prior is a module the node ran before this agent started, so it has something to back up. + prior map[string]bool + said map[string]string + retry chan []string +} + +type wantNow struct { + Want delivery.Want + Node bool // the node's own want + Present bool + Rev uint64 +} + +type compNow struct { + Raw []byte + C delivery.Composed + Dig string + Rev uint64 + Err error +} + +type attempt struct { + FP, Composed string + WantRev, CompRev uint64 + Held bool + AnsweredRev uint64 +} + +// New is an agent for a node. +func New(env Env) *Agent { + if env.Say == nil { + env.Say = func(string) {} + } + return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, + attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, + retry: make(chan []string, 8)} +} + +// Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition +// before any module's, then each change as it comes. +func (a *Agent) Run(ctx context.Context) error { + wants, err := a.env.Wants.Watch(ctx, "want."+a.env.Node+".>") + if err != nil { + return fmt.Errorf("the agent cannot watch its wants: %w", err) + } + comps, err := a.env.Composed.Watch(ctx, a.env.Node+".>") + if err != nil { + return fmt.Errorf("the agent cannot watch its compositions: %w", err) + } + var wantsCaught, compsCaught bool + dirty := map[string]bool{} + for { + select { + case <-ctx.Done(): + return nil + case modules := <-a.retry: + // Applied again though the keys did not change: a container that waited because it said busy (ADR 0360 + // §11) is replaced once it is idle. + a.mu.Lock() + for _, m := range modules { + delete(a.running, m) + delete(a.attempt, m) + dirty[m] = true + } + a.mu.Unlock() + case u, ok := <-wants: + if !ok { + return fmt.Errorf("the watch of the wants ended") + } + if u.Caught { + wantsCaught = true + } else if m := a.takeWant(u); m != "" { + dirty[m] = true + } + case u, ok := <-comps: + if !ok { + return fmt.Errorf("the watch of the compositions ended") + } + if u.Caught { + compsCaught = true + } else if m := a.takeComposition(u); m != "" { + dirty[m] = true + } + } + // Nothing is applied until what was there at the start is all in: a module's composition may arrive after its + // want, and the node's own composition comes before any module's. + if !(wantsCaught && compsCaught) || len(dirty) == 0 { + continue + } + a.settle(ctx, dirty) + dirty = map[string]bool{} + } +} + +// Retry asks for the modules to be applied again from the keys they have, though nothing changed: what ADR 0360 §11 +// needs for a container that waits because it said busy. It never blocks; a retry asked while eight wait is dropped, and +// asked again by its caller's next look. +func (a *Agent) Retry(modules ...string) { + select { + case a.retry <- modules: + default: + } +} + +// Running is the compositions the node runs now: each module's that the agent applied or found already running, and +// the node's own. What the node judges its health by and says of itself is built from them. +func (a *Agent) Running() []delivery.Composed { + a.mu.Lock() + defer a.mu.Unlock() + mods := make([]string, 0, len(a.running)) + for m := range a.running { + mods = append(mods, m) + } + sort.Strings(mods) + var out []delivery.Composed + for _, m := range mods { + h := a.running[m] + if h.Phase == delivery.PhaseRemoved { + continue + } + if c, ok := a.compositionFor(m, h.FP); ok && c.Err == nil && c.Dig == h.Composed { + out = append(out, c.C) + } + } + return out +} + +// takeWant takes a change of a want key and returns the module it concerns, or "" for a key that is not a want. +func (a *Agent) takeWant(u Update) string { + a.mu.Lock() + defer a.mu.Unlock() + if _, err := delivery.ParseWantNodeKey(u.Key); err == nil { + w := wantNow{Node: true, Rev: u.Revision} + if !u.Deleted { + nw, err := delivery.DecodeNodeWant(u.Value) + if err != nil { + a.sayOnce("want:node", fmt.Sprintf("the node's own want is not one: %v", err)) + return "" + } + w.Present, w.Want = true, delivery.Want{FP: "", Composed: nw.Composed} + } + a.wants[NodeModule] = w + return NodeModule + } + node, module, err := delivery.ParseWantKey(u.Key) + if err != nil || node != a.env.Node { + return "" + } + w := wantNow{Rev: u.Revision} + if !u.Deleted { + want, err := delivery.DecodeWant(u.Value) + if err != nil { + a.sayOnce("want:"+module, fmt.Sprintf("the want of %s is not one: %v", module, err)) + return "" + } + w.Present, w.Want = true, want + } + a.wants[module] = w + return module +} + +// takeComposition takes a change of a composition key and returns the module it concerns. +func (a *Agent) takeComposition(u Update) string { + node, module, _, isNode, err := delivery.ParseComposedKey(u.Key) + if err != nil || node != a.env.Node { + return "" + } + a.mu.Lock() + defer a.mu.Unlock() + if isNode { + module = NodeModule + } + if u.Deleted { + delete(a.comps, u.Key) + return module + } + now := compNow{Raw: u.Value, Rev: u.Revision} + c, err := delivery.DecodeComposed(u.Value) + if err == nil { + var trusted []ed25519.PublicKey + if a.env.Trusted != nil { + trusted = a.env.Trusted() + } + // The signature before anything of the composition is used: its digest is what the want names. + now.C = c + now.Dig, err = delivery.VerifyComposed(c, trusted) + if err != nil { + now.C = delivery.Composed{} + a.sayOnce("verify:"+u.Key, fmt.Sprintf("the composition %s is refused and nothing of it is applied: %v", u.Key, err)) + err = fmt.Errorf("refused: %w", err) + } + } else { + a.sayOnce("decode:"+u.Key, fmt.Sprintf("the composition %s cannot be read, and nothing of it is applied: %v", u.Key, err)) + } + now.Err = err + a.comps[u.Key] = now + return module +} + +func (a *Agent) sayOnce(key, line string) { + if a.said[key] == line { + return + } + a.said[key] = line + a.env.Say(line) +} + +// settle applies what the dirty modules' wants allow: the node's own first, then each module in name order. +func (a *Agent) settle(ctx context.Context, dirty map[string]bool) { + order := make([]string, 0, len(dirty)) + for m := range dirty { + order = append(order, m) + } + sort.Slice(order, func(i, j int) bool { + if (order[i] == NodeModule) != (order[j] == NodeModule) { + return order[i] == NodeModule + } + return order[i] < order[j] + }) + for _, m := range order { + if ctx.Err() != nil { + return + } + a.settleOne(ctx, m) + } +} + +func (a *Agent) compositionFor(module, fp string) (compNow, bool) { + var key string + var err error + if module == NodeModule { + key, err = delivery.ComposedNodeKey(a.env.Node) + } else { + key, err = delivery.ComposedKey(a.env.Node, module, fp) + } + if err != nil { + return compNow{}, false + } + c, ok := a.comps[key] + return c, ok +} + +// others are what the node's other modules run now, so what they hold is not taken: the compositions the agent runs and, +// for every module the agent does not hold yet, the resources the kept declaration has for it (ADR 0429 §4). +func (a *Agent) others(except string) []delivery.Composed { + var out []delivery.Composed + mods := make([]string, 0, len(a.running)) + for m := range a.running { + if m != except { + mods = append(mods, m) + } + } + sort.Strings(mods) + for _, m := range mods { + h := a.running[m] + if h.Phase == delivery.PhaseRemoved { + continue + } + if c, ok := a.compositionFor(m, h.FP); ok && c.Err == nil && c.Dig == h.Composed { + out = append(out, c.C) + } + } + if a.env.Kept != nil { + kept := a.env.Kept() + names := make([]string, 0, len(kept)) + for m := range kept { + if _, held := a.running[m]; !held && m != except { + names = append(names, m) + } + } + sort.Strings(names) + for _, m := range names { + c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]} + if m == NodeModule { + c.Module = "" + } + out = append(out, c) + } + } + return out +} + +// Everything is what the node runs: the compositions the agent runs, and the kept declaration's resources for each module +// the agent does not hold yet. What the node judges its health by, and what the filling, the take act and the judgement +// of directories read, is built from it. +func (a *Agent) Everything() []delivery.Composed { + a.mu.Lock() + defer a.mu.Unlock() + return a.others("\x00") +} + +// HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept +// declaration's. +func (a *Agent) HoldsNode() bool { + a.mu.Lock() + defer a.mu.Unlock() + h, ok := a.running[NodeModule] + return ok && h.Phase != delivery.PhaseRemoved +} + +func (a *Agent) settleOne(ctx context.Context, module string) { + a.mu.Lock() + w := a.wants[module] + a.mu.Unlock() + if !w.Present { + return + } + if w.Want.Removed { + a.remove(ctx, module, w) + return + } + a.mu.Lock() + comp, have := a.compositionFor(module, w.Want.FP) + a.mu.Unlock() + if !have || comp.Err != nil { + return // waits for the composition; a change of its key settles the module again + } + if comp.Dig != w.Want.Composed { + // A composition whose digest no want names is not applied. + a.mu.Lock() + a.sayOnce("digest:"+module, fmt.Sprintf("the composition of %s at %s has digest %s, and its want names %s; nothing is applied until they agree", + module, w.Want.FP, comp.Dig, w.Want.Composed)) + a.mu.Unlock() + return + } + a.mu.Lock() + if !a.checked[module] { + a.checked[module] = true + a.mu.Unlock() + if a.baseline(ctx, module, w, comp) { + return + } + } else { + a.mu.Unlock() + } + a.mu.Lock() + if r, ok := a.running[module]; ok && r.FP == w.Want.FP && r.Composed == w.Want.Composed && r.Phase != delivery.PhaseRemoved { + a.mu.Unlock() + return + } + att, tried := a.attempt[module] + same := tried && att.FP == w.Want.FP && att.Composed == w.Want.Composed + if same && !att.Held && att.WantRev == w.Rev && att.CompRev == comp.Rev { + a.mu.Unlock() + return // failed already for exactly these keys: retried by the next change of either + } + if same && att.Held && att.AnsweredRev >= w.Rev && att.CompRev == comp.Rev { + a.mu.Unlock() + return // held for a failed backup, and no answer since + } + others := a.others(module) + a.mu.Unlock() + a.apply(ctx, module, w, comp, others) +} + +// baseline decides whether what runs already is what the want names, once per module at the agent's start: the node's +// own have (what this node-engine last applied), else the declaration it kept before the switch (ADR 0539 §9). True when +// it is, with the have written; nothing is applied. +func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp compNow) bool { + if module != NodeModule { + if h, ok := a.env.Haves.Read(ctx, module); ok { + a.mu.Lock() + a.prior[module] = true + a.mu.Unlock() + if h.Composed != "" { + // This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A + // failed apply or a different composition is applied again. + if h.FP == w.Want.FP && h.Composed == w.Want.Composed && + (h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) { + a.mu.Lock() + a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} + a.mu.Unlock() + return true + } + return false + } + } + } + if a.env.Kept == nil { + return false + } + kept, ok := a.env.Kept()[module] + if ok { + a.mu.Lock() + a.prior[module] = true + a.mu.Unlock() + } + if !ok || !SameResources(kept, comp.C.Resources) { + return false + } + a.mu.Lock() + a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning} + a.mu.Unlock() + a.env.Haves.Write(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)}) + return true +} + +func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNow, others []delivery.Composed) { + fp, dig := w.Want.FP, w.Want.Composed + a.mu.Lock() + _, ranBefore := a.running[module] + ranBefore = ranBefore || a.prior[module] + a.mu.Unlock() + + // A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on + // without one (ADR 0539 §16). + if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip { + a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)}) + if got := a.env.Backup(ctx, module); !got.Done { + a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, got.Words)) + a.mu.Lock() + a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev} + prev := a.running[module] + a.mu.Unlock() + // It keeps running what it has: the have says what it holds and why it waits. + a.env.Haves.Write(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld), + Detail: got.Words}) + return + } + } + a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)}) + res := a.env.Apply(ctx, comp.C, others) + a.mu.Lock() + if res.Failed { + a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev} + a.mu.Unlock() + a.env.Say(fmt.Sprintf("applying %s at %s failed: %s", module, fp, res.Detail)) + a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail}) + return + } + delete(a.attempt, module) + a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning} + a.mu.Unlock() + a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail}) +} + +func (a *Agent) remove(ctx context.Context, module string, w wantNow) { + a.mu.Lock() + att := a.attempt[module] + r, ran := a.running[module] + if att.FP == "removed" && att.WantRev == w.Rev { + a.mu.Unlock() + return + } + if ran && r.Phase == delivery.PhaseRemoved { + a.mu.Unlock() + return + } + others := a.others(module) + a.mu.Unlock() + res := a.env.Remove(ctx, module, others) + a.mu.Lock() + if res.Failed { + a.attempt[module] = attempt{FP: "removed", WantRev: w.Rev} + a.mu.Unlock() + a.env.Say(fmt.Sprintf("removing %s failed: %s", module, res.Detail)) + return + } + delete(a.attempt, module) + a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved} + a.mu.Unlock() + a.env.Haves.Removed(ctx, module) +} + +// SameResources says whether two lists of resources hold the same resources, each by its id and its canonical JSON. +func SameResources(a, b []json.RawMessage) bool { + ma, ok := canonByID(a) + if !ok { + return false + } + mb, ok := canonByID(b) + if !ok || len(ma) != len(mb) { + return false + } + for id, x := range ma { + if y, there := mb[id]; !there || x != y { + return false + } + } + return true +} + +func canonByID(rs []json.RawMessage) (map[string]string, bool) { + out := map[string]string{} + for _, raw := range rs { + var head struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &head) != nil || head.ID == "" { + return nil, false + } + canon, err := Canonical(raw) + if err != nil { + return nil, false + } + out[head.ID] = string(canon) + } + return out, true +} + +// Canonical is raw with its object keys sorted and its spacing dropped, its numbers kept as written. +func Canonical(raw json.RawMessage) (json.RawMessage, error) { + var v any + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + if err := d.Decode(&v); err != nil { + return nil, err + } + return json.Marshal(v) +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go new file mode 100644 index 0000000..3b7608a --- /dev/null +++ b/internal/agent/agent_test.go @@ -0,0 +1,642 @@ +package agent + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const node = "n1" + +// The keys of the tests: the controller's and the operator's are trusted, a stranger's is not. +var controllerPub, controllerKey = newKey() +var operatorPub, operatorKey = newKey() +var _, strangerKey = newKey() + +func newKey() (ed25519.PublicKey, ed25519.PrivateKey) { + pub, key, err := ed25519.GenerateKey(nil) + if err != nil { + panic(err) + } + return pub, key +} + +func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } + +// A watcher the test writes to; it says Caught once the keys written before the watch began are out. +type feed struct { + mu sync.Mutex + ch chan Update + rev uint64 + first []Update +} + +func newFeed() *feed { return &feed{ch: make(chan Update, 256)} } + +func (f *feed) Watch(_ context.Context, _ string) (<-chan Update, error) { + f.mu.Lock() + defer f.mu.Unlock() + for _, u := range f.first { + f.ch <- u + } + f.ch <- Update{Caught: true} + return f.ch, nil +} + +// before writes a key as if it was there when the watch began. +func (f *feed) before(key string, value []byte) { + f.mu.Lock() + defer f.mu.Unlock() + f.rev++ + f.first = append(f.first, Update{Key: key, Value: value, Revision: f.rev}) +} + +func (f *feed) put(key string, value []byte) { + f.mu.Lock() + f.rev++ + r := f.rev + f.mu.Unlock() + f.ch <- Update{Key: key, Value: value, Revision: r} +} + +type haves struct { + mu sync.Mutex + stored map[string]Have + log []Have + removed []string +} + +func (h *haves) Read(_ context.Context, module string) (Have, bool) { + h.mu.Lock() + defer h.mu.Unlock() + x, ok := h.stored[module] + return x, ok +} +func (h *haves) Write(_ context.Context, x Have) { + h.mu.Lock() + defer h.mu.Unlock() + if h.stored == nil { + h.stored = map[string]Have{} + } + h.stored[x.Module] = x + h.log = append(h.log, x) +} +func (h *haves) Removed(_ context.Context, m string) { + h.mu.Lock() + defer h.mu.Unlock() + h.removed = append(h.removed, m) +} +func (h *haves) last(module string) Have { + h.mu.Lock() + defer h.mu.Unlock() + return h.stored[module] +} + +type bed struct { + t *testing.T + wants *feed + comps *feed + haves *haves + mu sync.Mutex + applied []string + removed []string + backups []string + failWith string + backup func(module string) Backup + kept map[string][]json.RawMessage + others [][]delivery.Composed + env Env +} + +func newBed(t *testing.T) *bed { + b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}} + b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves, + Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result { + b.mu.Lock() + defer b.mu.Unlock() + b.others = append(b.others, others) + name := c.Module + if name == "" { + name = NodeModule + } + b.applied = append(b.applied, name) + if b.failWith != "" { + return Result{Failed: true, Detail: b.failWith} + } + return Result{} + }, + Remove: func(_ context.Context, m string, _ []delivery.Composed) Result { + b.mu.Lock() + defer b.mu.Unlock() + b.removed = append(b.removed, m) + return Result{} + }, + Backup: func(_ context.Context, m string) Backup { + b.mu.Lock() + defer b.mu.Unlock() + b.backups = append(b.backups, m) + if b.backup != nil { + return b.backup(m) + } + return Backup{Done: true} + }, + Kept: func() map[string][]json.RawMessage { return b.kept }, + Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} }, + Say: func(s string) { t.Log(s) }, + } + return b +} + +func (b *bed) run() (stop func()) { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = New(b.env).Run(ctx) }() + return func() { cancel(); <-done } +} + +func file(module, id, content string) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id, + "content": content, "module": module}) + return raw +} + +// composition is a module's composition at a fingerprint: its key, its bytes and its digest, from the writer's code. +func composition(t *testing.T, module, fp string, data bool, rs ...json.RawMessage) (key string, raw []byte, digest string) { + t.Helper() + c := delivery.Composed{Node: node, Module: module, FP: fp, Data: data, Resources: rs} + var err error + if module == NodeModule { + c.Module, c.FP = "", "" + key, err = delivery.ComposedNodeKey(node) + } else { + key, err = delivery.ComposedKey(node, module, fp) + } + if err != nil { + t.Fatal(err) + } + return signedBy(t, c, key, controllerKey) +} + +// signedBy is c signed with signer, as the writer's code signs it: its key, its bytes and its digest. +func signedBy(t *testing.T, c delivery.Composed, key string, signer ed25519.PrivateKey) (string, []byte, string) { + t.Helper() + signed, err := delivery.SignComposed(c, signer) + if err != nil { + t.Fatal(err) + } + raw, err := delivery.EncodeComposed(signed) + if err != nil { + t.Fatal(err) + } + digest, err := delivery.ComposedDigestOf(signed) + if err != nil { + t.Fatal(err) + } + return key, raw, digest +} + +func wantFor(t *testing.T, module, fp, digest, backup string) (string, []byte) { + t.Helper() + key, err := delivery.WantKey(node, module) + if err != nil { + t.Fatal(err) + } + raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup}) + if err != nil { + t.Fatal(err) + } + return key, raw +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatal(what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func (b *bed) appliedList() []string { + b.mu.Lock() + defer b.mu.Unlock() + return append([]string(nil), b.applied...) +} + +// R6-A AC1 (the agent, with a bus that is a fake): given a want naming a digest and that composition, the module is +// applied and its have names the fingerprint and the digest; a composition whose digest no want names is not applied. +func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) { + b := newBed(t) + stop := b.run() + defer stop() + + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + // A composition no want names yet is not applied. + b.comps.put(ck, craw) + time.Sleep(100 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("a composition no want names was applied: %v", got) + } + wk, wraw := wantFor(t, "web", fp, dig, "") + b.wants.put(wk, wraw) + eventually(t, "the module was not applied", func() bool { return len(b.appliedList()) == 1 }) + eventually(t, "the have does not name both", func() bool { + h := b.haves.last("web") + return h.FP == fp && h.Composed == dig && h.Phase == string(delivery.PhaseRunning) + }) + + // A want naming a digest the composition at that fingerprint does not have: nothing is applied. + fp2 := fpOf("b") + ck2, craw2, _ := composition(t, "web", fp2, false, file("web", "x", "2")) + b.comps.put(ck2, craw2) + wk2, wraw2 := wantFor(t, "web", fp2, "sha256:"+strings.Repeat("0", 64), "") + b.wants.put(wk2, wraw2) + time.Sleep(100 * time.Millisecond) + if got := b.appliedList(); len(got) != 1 { + t.Fatalf("a want naming another digest applied the composition: %v", got) + } +} + +// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied +// and every have names its fingerprint and digest. +func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + rs := []json.RawMessage{file("web", "x", "1"), file("web", "y", "2")} + ck, craw, dig := composition(t, "web", fp, false, rs...) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + // The kept declaration holds the same resources, in another order and spacing. + b.kept = map[string][]json.RawMessage{"web": {json.RawMessage(strings.ReplaceAll(string(rs[1]), ",", " , ")), rs[0]}} + stop := b.run() + defer stop() + eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig }) + if h := b.haves.last("web"); h.FP != fp || h.Phase != string(delivery.PhaseRunning) { + t.Fatalf("the have: %+v", h) + } + time.Sleep(50 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("a module that already runs what the want names was applied: %v", got) + } +} + +// What the node-engine itself last applied is what runs after a restart, not the kept declaration. +func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}} + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + stop := b.run() + defer stop() + time.Sleep(150 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("a module that runs what the want names was applied after a restart: %v", got) + } +} + +// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change +// of either key. +func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) { + b := newBed(t) + b.failWith = "web.x: no space" + stop := b.run() + defer stop() + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.comps.put(ck, craw) + b.wants.put(wk, wraw) + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + eventually(t, "the have does not say failed", func() bool { + h := b.haves.last("web") + return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp + }) + // An unrelated key changes: not a retry. + ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9")) + b.comps.put(ck9, craw9) + time.Sleep(150 * time.Millisecond) + if n := len(b.appliedList()); n != 1 { + t.Fatalf("a failed apply was retried by an unrelated key (%d applies)", n) + } + // The want is written again: the retry. + b.mu.Lock() + b.failWith = "" + b.mu.Unlock() + b.wants.put(wk, wraw) + eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 }) + eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) }) +} + +// R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is +// applied and the have says held; when the want then says skip, the module is applied. +func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { + b := newBed(t) + fp1, fp2 := fpOf("a"), fpOf("b") + ck1, craw1, dig1 := composition(t, "db", fp1, true, file("db", "x", "1")) + wk1, wraw1 := wantFor(t, "db", fp1, dig1, "") + b.comps.before(ck1, craw1) + b.wants.before(wk1, wraw1) + b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}} + refuse := true + b.backup = func(string) Backup { + if refuse { + return Backup{Words: "restic: the repository is locked"} + } + return Backup{Done: true} + } + stop := b.run() + defer stop() + eventually(t, "baseline", func() bool { return b.haves.last("db").Composed == dig1 }) + + ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2")) + b.comps.put(ck2, craw2) + wk2, wraw2 := wantFor(t, "db", fp2, dig2, "") + b.wants.put(wk2, wraw2) + eventually(t, "the backup was not asked", func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.backups) == 1 + }) + eventually(t, "the have does not say held", func() bool { + h := b.haves.last("db") + return h.Phase == string(delivery.PhaseHeld) && strings.Contains(h.Detail, "repository is locked") && h.FP == fp1 + }) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("something was applied behind a failed backup: %v", got) + } + + // The answer arrives in the module's own want: back up again (it fails again), then skip. + wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain) + b.wants.put(wk, again) + eventually(t, "back up again did not ask the backup", func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.backups) == 2 + }) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied behind a second failed backup: %v", got) + } + _, skip := wantFor(t, "db", fp2, dig2, delivery.BackupSkip) + b.wants.put(wk, skip) + eventually(t, "skip did not apply the module", func() bool { return len(b.appliedList()) == 1 }) + b.mu.Lock() + n := len(b.backups) + b.mu.Unlock() + if n != 2 { + t.Fatalf("a backup was asked after the answer was to skip: %d asks", n) + } + eventually(t, "not running at the new fingerprint", func() bool { + h := b.haves.last("db") + return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2 + }) +} + +// A data module with a good backup is applied after it; one that never ran has nothing to back up. +func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) { + b := newBed(t) + stop := b.run() + defer stop() + fp := fpOf("c") + ck, craw, dig := composition(t, "db", fp, true, file("db", "x", "1")) + wk, wraw := wantFor(t, "db", fp, dig, "") + b.comps.put(ck, craw) + b.wants.put(wk, wraw) + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + b.mu.Lock() + n := len(b.backups) + b.mu.Unlock() + if n != 0 { + t.Fatalf("a module that never ran was backed up (%d)", n) + } + fp2 := fpOf("d") + ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2")) + wk2, wraw2 := wantFor(t, "db", fp2, dig2, "") + b.comps.put(ck2, craw2) + b.wants.put(wk2, wraw2) + eventually(t, "second not applied", func() bool { return len(b.appliedList()) == 2 }) + b.mu.Lock() + n = len(b.backups) + b.mu.Unlock() + if n != 1 { + t.Fatalf("a module that ran before was backed up %d times, want once", n) + } +} + +// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed. +func TestARemovedWantRemovesThatModuleOnly(t *testing.T) { + b := newBed(t) + stop := b.run() + defer stop() + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + b.comps.put(ck, craw) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.wants.put(wk, wraw) + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + gone, _ := delivery.EncodeWant(delivery.Want{Removed: true}) + b.wants.put(wk, gone) + eventually(t, "not removed", func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.removed) == 1 && b.removed[0] == "web" + }) + eventually(t, "the have was not written removed", func() bool { + b.haves.mu.Lock() + defer b.haves.mu.Unlock() + return len(b.haves.removed) == 1 + }) +} + +// The node's own composition is applied before any module's, whichever key came first. +func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + nk, nraw, ndig := composition(t, NodeModule, "", false) + nwk, err := delivery.WantNodeKey(node) + if err != nil { + t.Fatal(err) + } + nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig}) + if err != nil { + t.Fatal(err) + } + b.comps.before(ck, craw) + b.comps.before(nk, nraw) + b.wants.before(wk, wraw) + b.wants.before(nwk, nwraw) + stop := b.run() + defer stop() + eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 }) + if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" { + t.Fatalf("the order was %v, the node's own first", got) + } +} + +// R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was +// signed and one unsigned are refused and nothing of them is applied (ADR 0539 §3). +func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) { + b := newBed(t) + stop := b.run() + defer stop() + + plain := func(module, fp, content string) delivery.Composed { + return delivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{file(module, "x", content)}} + } + put := func(c delivery.Composed, signer ed25519.PrivateKey, tamper func([]byte) []byte) { + key, _ := delivery.ComposedKey(node, c.Module, c.FP) + var raw []byte + var dig string + if signer == nil { + raw, _ = delivery.EncodeComposed(c) + dig, _ = delivery.ComposedDigestOf(c) + } else { + _, raw, dig = signedBy(t, c, key, signer) + } + if tamper != nil { + raw = tamper(raw) + } + b.comps.put(key, raw) + wk, wraw := wantFor(t, c.Module, c.FP, dig, "") + b.wants.put(wk, wraw) + } + + // Refused: signed by a stranger, changed after signing, unsigned. + put(plain("stranger", fpOf("1"), "x"), strangerKey, nil) + put(plain("changed", fpOf("2"), "before"), controllerKey, func(raw []byte) []byte { + return []byte(strings.Replace(string(raw), "before", "after!", 1)) + }) + put(plain("unsigned", fpOf("3"), "x"), nil, nil) + time.Sleep(200 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("a composition that is not signed by a trusted key was applied: %v", got) + } + + // Applied: signed by the controller, signed by the operator. + put(plain("bycontroller", fpOf("4"), "x"), controllerKey, nil) + put(plain("byoperator", fpOf("5"), "x"), operatorKey, nil) + eventually(t, "the trusted signers' compositions were not applied", func() bool { return len(b.appliedList()) == 2 }) + got := b.appliedList() + if !(got[0] == "bycontroller" && got[1] == "byoperator") && !(got[0] == "byoperator" && got[1] == "bycontroller") { + t.Fatalf("applied %v", got) + } + if h := b.haves.last("byoperator"); h.Phase != string(delivery.PhaseRunning) || h.FP != fpOf("5") { + t.Fatalf("the operator's module has no running have: %+v", h) + } +} + +// With no key trusted, nothing is applied: a node that does not know its signers fails closed. +func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) { + b := newBed(t) + b.env.Trusted = nil + stop := b.run() + defer stop() + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + b.comps.put(ck, craw) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.wants.put(wk, wraw) + time.Sleep(200 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied with no trusted key: %v", got) + } +} + +// What a module's apply must not take is every other module that runs, those the agent holds and those still running from +// the kept declaration (ADR 0429 §4): the resources of the kept declaration reach Apply as the others. +func TestApplyingOneModuleHoldsOutTheModulesStillRunningFromTheKeptDeclaration(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, NodeModule: {file("node", "n", "n")}} + stop := b.run() + defer stop() + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + b.comps.put(ck, craw) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.wants.put(wk, wraw) + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + b.mu.Lock() + defer b.mu.Unlock() + var names []string + for _, c := range b.others[0] { + names = append(names, c.Module) + } + if len(names) != 2 || !strings.Contains(strings.Join(names, ","), "db") { + t.Fatalf("the others of web were %v: db and the node's own run from the kept declaration", names) + } +} + +// Everything is what runs: the agent's compositions and the kept resources of each module it does not hold. +func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, "web": {file("web", "x", "old")}} + a := New(b.env) + if got := a.Everything(); len(got) != 2 { + t.Fatalf("before the agent holds anything: %d compositions, want the 2 kept", len(got)) + } + a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: "sha256:" + strings.Repeat("1", 64), Phase: delivery.PhaseRunning} + ck, craw, dig := composition(t, "web", fpOf("a"), false, file("web", "x", "new")) + a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: dig, Phase: delivery.PhaseRunning} + a.takeComposition(Update{Key: ck, Value: craw, Revision: 1}) + got := a.Everything() + if len(got) != 2 { + t.Fatalf("%d compositions", len(got)) + } + for _, c := range got { + if c.Module == "web" && !strings.Contains(string(c.Resources[0]), "new") { + t.Fatal("web is the kept declaration's, not the composition the agent runs") + } + } + if a.HoldsNode() { + t.Fatal("the agent holds no node composition") + } +} + +// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this +// node-engine wrote its haves, they are what it knows (the kept declaration is only for the first start after the switch). +func TestAFailedApplyIsAppliedAgainAfterARestartWhateverIsKept(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}} + b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed)}} + stop := b.run() + defer stop() + eventually(t, "a failed apply was marked running from the kept declaration", func() bool { return len(b.appliedList()) == 1 }) +} + +// A have written before the switch carries no composition: then the kept declaration says what runs. +func TestAHaveWithNoCompositionDefersToTheKeptDeclaration(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}} + b.haves.stored = map[string]Have{"web": {Module: "web", Phase: "running"}} + stop := b.run() + defer stop() + eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig }) + time.Sleep(50 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied %v", got) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index a22e345..03c57b6 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -21,7 +21,6 @@ import ( "os" "os/exec" "path/filepath" - "slices" "sort" "strconv" "strings" @@ -300,6 +299,60 @@ func ApplyMindingWindows( unseal Unseal, keep Keep, windows *Windows, +) (Report, store.State, error) { + return applyScoped(ctx, sys, d, nil, known, origin, run, log, unseal, keep, windows) +} + +// Scope is what makes an apply one module's (novox/hq ADR 0429 §4, ADR 0539 §9): the module whose want changed, and +// the resources the node's other modules hold, so a path, a unit or a group another module still asks for is neither +// removed nor taken from it. Only that module's own records are orphans; a removed module is a Scope with an empty +// declaration. +type Scope struct { + // Module is the module applied. NodeModule (store.NodeModule) is the node's own composition. + Module string + // Others are the resources of every other module the node has a composition for. + Others []declaration.Resource +} + +// owns reports whether a record belongs to the scope's module. +func (sc *Scope) owns(a store.Applied) bool { + return store.OwnerOf(a, declaration.ModuleGuess) == sc.Module +} + +// isNode is whether the scope is the node's own composition, the one that carries identities, groups, libraries and +// the adoption. +func (sc *Scope) isNode() bool { return sc == nil || sc.Module == store.NodeModule } + +// ApplyModule makes the machine match one module's declaration, and only that module's: everything else the node +// runs is left as it is. d holds the module's resources; scope names the module and what the others hold. +func ApplyModule( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + scope Scope, + known store.State, + origin string, + run Runner, + log func(string), + unseal Unseal, + keep Keep, + windows *Windows, +) (Report, store.State, error) { + return applyScoped(ctx, sys, d, &scope, known, origin, run, log, unseal, keep, windows) +} + +func applyScoped( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + scope *Scope, + known store.State, + origin string, + run Runner, + log func(string), + unseal Unseal, + keep Keep, + windows *Windows, ) (report Report, _ store.State, _ error) { // Said whichever way the apply ended, failure included: a window being open is a fact about the // machine, not about this apply. @@ -326,7 +379,9 @@ func ApplyMindingWindows( } // What an unfinished apply found a unit as is kept only while its service is declared: one // declared again later is read afresh, as any resource with no record is (novox/hq ADR 0118). - known.DropFoundUndeclared(declared, origin) + if scope == nil { + known.DropFoundUndeclared(declared, origin) + } // Which units the mesh made, read before any removal can take a unit file's record away — so // whichever order a module declared a unit and its file in, the unit is known for the mesh's @@ -351,7 +406,11 @@ func ApplyMindingWindows( // it back would stop or disable a unit the declaration says is running, every container on it // with it, only for the declared one to start it again in the same apply. The record is // forgotten; the unit's found state is the remaining record's to give back. - heldUnits := unitsHeld(d.Resources) + everything := d.Resources + if scope != nil { + everything = append(append([]declaration.Resource{}, d.Resources...), scope.Others...) + } + heldUnits := unitsHeld(everything) // **A whole file still declared at its path by another resource is handed to it, never removed // first** (novox/hq ADR 0223). A file can change owners between modules — the machine's resolver @@ -361,13 +420,13 @@ func ApplyMindingWindows( // and stayed so until the new resource's turn came, with every resource in between applied on a // machine without it. The record going is kept until the one declaring the path now is recorded // in its place: then it is forgotten, and what it kept of the original goes with the file. - heldFiles := filesHeld(d.Resources) + heldFiles := filesHeld(everything) handedFiles := map[string]store.Applied{} // **A group still asked for by another resource is not given back when one record of it goes** // (novox/hq ADR 0252), on the unit's rule above: two modules may each want the account in one group, // and the one going takes nothing the other still needs. - wanted := groupsWanted(d.Resources) + wanted := groupsWanted(everything) // And what a record going hands to a resource that still asks for its group (novox/hq ADR 0252 §3). handed := Handed{} @@ -375,6 +434,18 @@ func ApplyMindingWindows( var action, detail string var err error if by, held := heldFiles[fileKey(orphan)]; held && by != orphan.ID { + if scope != nil && !declared[by] { + // Another module declares the path: it is that module's, so this record is let go and the file is + // neither removed nor taken from it (ADR 0539 §9). + known.Forget(orphan.ID) + detail := "no longer declared; " + by + " of another module still holds the path, so it was left as it is" + report.Outcomes = append(report.Outcomes, Outcome{ + ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, + Action: "forgotten", Detail: detail, + }) + log(fmt.Sprintf(" forgotten %s (%s): %s", orphan.ID, orphan.Target, detail)) + return nil + } handedFiles[by] = orphan return nil } @@ -482,20 +553,11 @@ func ApplyMindingWindows( } handover := map[string][]store.Applied{} for _, orphan := range known.Orphans(declared, origin) { - if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) { - protecting = append(protecting, orphan) + if scope != nil && !scope.owns(orphan) { continue } - // **A module the mesh left out is not a module the mesh removed** (novox/hq ADR 0163, rule - // 6): its resources are absent because a setting stored for it cannot compose, and the - // mesh said so by name. What the host wrote for it stays as it is, recorded, until the - // module is declared again or unassigned. - if module, left := d.LeftOutModuleOf(orphan.ID); left { - report.Outcomes = append(report.Outcomes, Outcome{ - ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, - Action: "unchanged", Detail: "kept: " + module + " was left out of this declaration by the mesh, not removed", - }) - log(fmt.Sprintf(" kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module)) + if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) { + protecting = append(protecting, orphan) continue } if by, replaced := replacedBy[orphan.ID]; replaced { @@ -538,12 +600,7 @@ func ApplyMindingWindows( // what is assigned: a carried bundle's silence is not an unassignment. if origin == store.OriginDeclared { for _, h := range append([]store.Held{}, known.Held...) { - if declared[h.ID] { - continue - } - if slices.Contains(d.LeftOut, h.Module) { - // Left out, not unassigned (ADR 0163, rule 6): still held for the module, as the - // mesh asked. + if declared[h.ID] || (scope != nil && h.Module != scope.Module) { continue } known.Release(h.ID) @@ -745,7 +802,7 @@ func ApplyMindingWindows( in.held = heldRoots(verdicts, d.Libraries) // A tool runner no longer put in a group leaves it before anything is applied, so its unit, which names the // change under restart-on, is restarted in this same apply to lose it (ADR 0328 §11, ADR 0334 §2, ADR 0252 §3). - if origin == store.OriginDeclared { + if origin == store.OriginDeclared && scope.isNode() { for _, o := range leaveRunners(ctx, sys, d.Libraries, d.Identities, &known, run, wanted) { if o.Action == "restored" { changed[o.changes] = true @@ -1037,6 +1094,11 @@ func ApplyMindingWindows( } // Only now. The record follows the fact, never leads it. recordApplied(&known, origin, resource, outcome, kept, keptMode, keptOwner) + if scope != nil { + known.SetModule(resource.Identity(), scope.Module) + } else if module, ours := d.ModuleOfID(resource.Identity()); ours { + known.SetModule(resource.Identity(), module) + } appliedOK[resource.Identity()] = true // Its module has been taken, and what was held for it is now the mesh's. A file written // into, or a service whose lifecycle is the machine's, replaced nothing that was found, so @@ -1139,7 +1201,7 @@ func ApplyMindingWindows( } // Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder. - if origin == store.OriginDeclared { + if origin == store.OriginDeclared && scope.isNode() { // And an identity, then a library group, no longer declared goes, once what ran as it went above (ADR 0328 // §3, §4). if tool, err := toolFor(sys); err == nil { @@ -1158,7 +1220,7 @@ func ApplyMindingWindows( // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100), and on every // converged apply, not once (ADR 0168). Skipped, it is said: a step that does nothing is never // silent (issue 143). - if len(failures) == 0 { + if len(failures) == 0 && scope.isNode() { did, err := retireFirewall(ctx, d, origin, &known, run, log) if err != nil { return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} @@ -1169,7 +1231,7 @@ func ApplyMindingWindows( return report, known, err } } - } else if rec := known.Firewall; origin == store.OriginDeclared && d.Adoption == nil && rec != nil && + } else if rec := known.Firewall; len(failures) > 0 && scope.isNode() && origin == store.OriginDeclared && d.Adoption == nil && rec != nil && rec.Kind == string(firewall.UFW) && rec.WasActive && firewall.Active(ctx, run) { report.Firewall = fmt.Sprintf("left in force: %d resource(s) failed, and the found firewall is retired only after a clean apply", len(failures)) log(" kept ufw in force: " + report.Firewall) diff --git a/internal/apply/left_out_test.go b/internal/apply/containers_kept_test.go similarity index 69% rename from internal/apply/left_out_test.go rename to internal/apply/containers_kept_test.go index a7b11a5..3bc8684 100644 --- a/internal/apply/left_out_test.go +++ b/internal/apply/containers_kept_test.go @@ -75,69 +75,6 @@ func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) { var specOfLast string -// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163, -// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held. -// A module simply absent is removed as it always was. -func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) { - var removed []string - gone := map[string]bool{} - run := func(_ context.Context, name string, args ...string) (string, error) { - args = oldShape(name, args) - if name != "docker" { - return "", nil - } - switch args[0] { - case "info": - return "29.0.0\n", nil - case "rm": - removed = append(removed, args[len(args)-1]) - gone[args[len(args)-1]] = true - case "container": - if gone[args[len(args)-1]] { - return "", errors.New("no such container") - } - return "true\tspec", nil - } - return "", nil - } - known := store.State{ - Resources: []store.Applied{ - {ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared}, - {ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared}, - }, - Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}}, - } - d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[ - {"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"} - ]}`) - report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) - if err != nil { - t.Fatal(err) - } - if len(removed) != 1 || removed[0] != "old" { - t.Fatalf("removed %v; only the module that is absent goes", removed) - } - if _, kept := state.At("container", "web"); !kept { - t.Fatal("the left-out module's record was forgotten") - } - if _, held := state.HeldAt("web.page"); !held { - t.Fatal("the left-out module's hold was released") - } - said := false - for _, o := range report.Outcomes { - if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") { - said = true - } - if o.ID == "web.server" && o.Action != "unchanged" { - t.Errorf("the left-out module's container was %s", o.Action) - } - } - if !said { - t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes) - } -} - -// A container's capabilities reach the runtime and are part of its spec (novox/hq ADR 0170). func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) { var ran []string run := func(_ context.Context, name string, args ...string) (string, error) { diff --git a/internal/apply/module_scope_test.go b/internal/apply/module_scope_test.go new file mode 100644 index 0000000..2ad9d20 --- /dev/null +++ b/internal/apply/module_scope_test.go @@ -0,0 +1,113 @@ +package apply + +import ( + "context" + "path/filepath" + "testing" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// R6-A AC3: given two modules declaring one path, when one module's want changes, the shared path is neither removed +// nor taken from the other module (novox/hq ADR 0539 §9). +func TestASharedPathIsNeitherRemovedNorTakenFromTheOtherModule(t *testing.T) { + dir := t.TempDir() + shared := filepath.Join(dir, "shared.conf") + own := filepath.Join(dir, "a-own.conf") + a := declare(t, `{"id":"a.shared","type":"file","path":"`+shared+`","content":"x\n","module":"a"}, + {"id":"a.own","type":"file","path":"`+own+`","content":"a\n","module":"a"}`) + b := declare(t, `{"id":"b.shared","type":"file","path":"`+shared+`","content":"x\n","module":"b"}`) + + state := store.State{} + var err error + _, state, err = ApplyModule(context.Background(), archHost(t), a, Scope{Module: "a", Others: b.Resources}, + state, store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + _, state, err = ApplyModule(context.Background(), archHost(t), b, Scope{Module: "b", Others: a.Resources}, + state, store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + + // A's want changes: it no longer declares the shared path, and b still does. + a2 := declare(t, `{"id":"a.own","type":"file","path":"`+own+`","content":"a\n","module":"a"}`) + report, state, err := ApplyModule(context.Background(), archHost(t), a2, Scope{Module: "a", Others: b.Resources}, + state, store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !exists(shared) { + t.Fatal("the path another module still declares was removed") + } + if _, still := state.Find("b.shared"); !still { + t.Fatal("the other module's record was taken away") + } + if _, still := state.Find("a.shared"); still { + t.Fatal("the module's own record of the shared path was kept") + } + if o := outcomeOf(report, "a.shared"); o.Action != "forgotten" { + t.Fatalf("the shared path: %+v", o) + } +} + +// R6-A AC4: given modules A and B on a node, when A's want is removed, only A's resources are removed and B is +// untouched. +func TestARemovedModuleTakesOnlyItsOwnResources(t *testing.T) { + dir := t.TempDir() + fa, fb := filepath.Join(dir, "a.conf"), filepath.Join(dir, "b.conf") + d := declare(t, `{"id":"a.conf","type":"file","path":"`+fa+`","content":"a\n","module":"a"}, + {"id":"b.conf","type":"file","path":"`+fb+`","content":"b\n","module":"b"}`) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if rec, _ := state.Find("a.conf"); rec.Module != "a" { + t.Fatalf("the owner was not recorded: %+v", rec) + } + + gone := &declaration.Declaration{} + b := declare(t, `{"id":"b.conf","type":"file","path":"`+fb+`","content":"b\n","module":"b"}`) + report, state, err := ApplyModule(context.Background(), archHost(t), gone, Scope{Module: "a", Others: b.Resources}, + state, store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if exists(fa) || !exists(fb) { + t.Fatalf("a's file exists %v, b's file exists %v", exists(fa), exists(fb)) + } + if o := outcomeOf(report, "a.conf"); o.Action != "removed" { + t.Fatalf("a's file: %+v", o) + } + if _, still := state.Find("b.conf"); !still { + t.Fatal("b's record was removed") + } + if _, still := state.Find("a.conf"); still { + t.Fatal("a's record is still there") + } +} + +// A record from before the owner was kept is read by its id, so the first removal after the switch is the module's too. +func TestARecordWithoutAnOwnerBelongsToTheModuleItsIdNames(t *testing.T) { + dir := t.TempDir() + fa, fb := filepath.Join(dir, "a.conf"), filepath.Join(dir, "b.conf") + d := declare(t, `{"id":"a.conf","type":"file","path":"`+fa+`","content":"a\n"}, + {"id":"b.conf","type":"file","path":"`+fb+`","content":"b\n"}`) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + for i := range state.Resources { + state.Resources[i].Module = "" + } + _, state, err = ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, Scope{Module: "a"}, + state, store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if exists(fa) || !exists(fb) { + t.Fatalf("a's file exists %v, b's file exists %v", exists(fa), exists(fb)) + } +} diff --git a/internal/apply/schedule.go b/internal/apply/schedule.go index 9b7146b..f40ee63 100644 --- a/internal/apply/schedule.go +++ b/internal/apply/schedule.go @@ -127,6 +127,17 @@ func (s *Scheduler) RecordWindowsIn(w *Windows) { // work on the predecessor's data, under the predecessor's service, which is the one thing an // adopted node must not do. Nil on a converged node, where nothing is held. func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { + s.sync(d, held, nil) +} + +// SyncModule is Sync for one module's declaration (novox/hq ADR 0539 §9): the schedules of that module are armed, +// re-armed or forgotten, and the schedules of every other module are left as they are. owns says whether a scheduled +// step's id is the module's. +func (s *Scheduler) SyncModule(d *declaration.Declaration, held map[string]bool, owns func(id string) bool) { + s.sync(d, held, owns) +} + +func (s *Scheduler) sync(d *declaration.Declaration, held map[string]bool, owns func(id string) bool) { s.mu.Lock() defer s.mu.Unlock() @@ -177,7 +188,7 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { } for id := range s.jobs { - if !seen[id] { + if !seen[id] && (owns == nil || owns(id)) { delete(s.jobs, id) } } diff --git a/internal/link/asked_test.go b/internal/link/asked_test.go deleted file mode 100644 index 8211c4b..0000000 --- a/internal/link/asked_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package link - -import ( - "context" - "strings" - "testing" - "time" -) - -// The `report` verb (novox/hq to-be 45 §6), what healer H1 asks a machine whose send went unreported: -// the node says again what it last applied, as an ordinary report through the one queue. - -// **Asked, the reconcile's account is said even when it is not news** — the mesh asked because it does -// not know, and silence would be the very fault it is asking about. -func TestAskedToReportSaysWhatWasLastAppliedEvenWhenNotNews(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - var lines []string - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) - q.Say = func(s string) { lines = append(lines, s) } - q.Reconcile = func(context.Context) (Report, bool) { - return Report{Declared: "d-kept", Order: Order{Epoch: 7, Sequence: 12}, Applied: []string{"a"}}, true - } - q.News = func(Report) bool { return false } - q.attach(context.Background(), l) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - - q.ReconcileDue() // an ordinary reconcile: not news, not said - time.Sleep(50 * time.Millisecond) - if n := len(l.said()); n != 0 { - t.Fatalf("an ordinary reconcile that is not news said %d report(s)", n) - } - q.ReportAsked() - eventually(t, "asked, the node said nothing", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - r := l.said()[0] - if r.Declared != "d-kept" || r.Sequence != 12 || r.Epoch != 7 || r.ReportSequence == 0 || r.Node != m.Node { - t.Fatalf("the account said is not the one kept, in order: %+v", r) - } - if !strings.Contains(strings.Join(lines, "\n"), "asked by the mesh") { - t.Errorf("being asked is not said in the journal: %q", lines) - } -} - -// **A delivery waiting answers the question better**: it is applied, once, and its report is the answer. -func TestAskedWhileADeliveryWaitsIsThatDeliverysReport(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - reconciled := 0 - q.Reconcile = func(context.Context) (Report, bool) { reconciled++; return Report{}, true } - q.attach(context.Background(), l) - q.Deliver(ordered(t, key, "new", 7, 13)) - q.ReportAsked() - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - time.Sleep(50 * time.Millisecond) - stop() - <-worker - if reconciled != 0 || len(l.said()) != 1 || l.said()[0].Sequence != 13 { - t.Fatalf("reconciled %d, said %+v", reconciled, l.said()) - } -} - -// askingLink is a quiet link the mesh can ask. -type askingLink struct { - *quietLink - asks chan struct{} -} - -func (l askingLink) AskedToReport() <-chan struct{} { return l.asks } - -// **The link hands the question to the queue**: asked on the link, the account is said on it. -func TestALinkThatIsAskedHandsItToTheQueue(t *testing.T) { - m, _ := aMember(t) - l := askingLink{quietLink: newQuietLink(), asks: make(chan struct{}, 1)} - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) - q.Reconcile = func(context.Context) (Report, bool) { return Report{Declared: "d-kept"}, true } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - worker := runWorker(ctx, q) - served := make(chan error, 1) - go func() { served <- serve(ctx, l, m, q, nil, time.Second) }() - l.asks <- struct{}{} - eventually(t, "asked on the link, nothing was said", func() bool { - for _, r := range l.said() { - if r.Declared == "d-kept" { - return true - } - } - return false - }) - stop() - <-served - <-worker -} - -// **Against a real server**: the question on the node's own subject reaches it, and another's does not. -func TestNatsTheMeshAsksANodeToReport(t *testing.T) { - conn, js := aBus(t) - l := &natsLink{conn: conn, js: js, node: "asked"} - l.hearAsks() - defer l.Close() - if err := conn.Publish(AskReportSubject("another"), []byte(`{}`)); err != nil { - t.Fatal(err) - } - if err := conn.Publish(AskReportSubject("asked"), []byte(`{"by":"the controller's healer H1"}`)); err != nil { - t.Fatal(err) - } - if err := conn.Flush(); err != nil { - t.Fatal(err) - } - select { - case <-l.AskedToReport(): - case <-time.After(5 * time.Second): - t.Fatal("the question on the node's own subject never reached it") - } - select { - case <-l.AskedToReport(): - t.Fatal("asked twice: another node's question reached this one") - case <-time.After(200 * time.Millisecond): - } - if AskReportSubject("asked") != "mesh.node.asked.ask.report" { - t.Fatalf("the subject %q is not the one the controller asks on and grants", AskReportSubject("asked")) - } -} diff --git a/internal/link/asktool_nats_test.go b/internal/link/asktool_nats_test.go index ca34dbd..7e266cf 100644 --- a/internal/link/asktool_nats_test.go +++ b/internal/link/asktool_nats_test.go @@ -25,8 +25,8 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) { t.Fatal(err) } - q := &Queue{Membership: Membership{Node: node}} - q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node}) + q := &Session{Membership: Membership{Node: node}} + q.Attach(t.Context(), &natsLink{conn: conn, js: js, node: node}) ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() a, err := q.AskTool(ctx, "sensors", "sensors_health") diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index eaa284f..3442a33 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -184,7 +184,7 @@ func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h } func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { m, _ := aMember(t) l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)} - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) + q := &Session{Membership: m, Timeout: time.Second} q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil } ctx, stop := context.WithCancel(context.Background()) served := make(chan error, 1) diff --git a/internal/link/have.go b/internal/link/have.go index 6428a36..caa01c1 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -37,6 +37,8 @@ type haveState struct { modules map[string]ModuleFact health map[string]delivery.Health node *delivery.HaveNode + // nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts. + nodeComposed string // written is the build of each module this node-engine wrote a have key for and that the declaration it applied // still holds; removed is each such module the latest declaration no longer holds, with the build its key last // said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted). @@ -97,11 +99,23 @@ func nodeFacts(r Report) delivery.HaveNode { } // noteApply takes what an apply's report says of the modules and the node. -func (q *Queue) noteApply(r Report) bool { - if r.Modules == nil { - return false - } +func (q *Session) noteApply(r Report) bool { s := &q.haves + if r.Modules == nil { + // Facts that say nothing of any module (the node agent writes those, ADR 0539 §9): the node's own key and the + // health the facts carry. + if r.Refused != "" { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if r.Health != nil { + s.health = moduleHealths(r.Health) + } + n := nodeFacts(r) + s.node = &n + return true + } s.mu.Lock() defer s.mu.Unlock() next := map[string]ModuleFact{} @@ -146,7 +160,7 @@ func (q *Queue) noteApply(r Report) bool { } // noteHealth takes a look's health, and says whether any module's changed. -func (q *Queue) noteHealth(h Health) bool { +func (q *Session) noteHealth(h Health) bool { s := &q.haves s.mu.Lock() defer s.mu.Unlock() @@ -170,7 +184,7 @@ func (q *Queue) noteHealth(h Health) bool { } // sayHave is a line in the node-engine's log, if it has one. -func (q *Queue) sayHave(s string) { +func (q *Session) sayHave(s string) { if q.Say != nil { q.Say(s) } @@ -180,7 +194,7 @@ func (q *Queue) sayHave(s string) { // §4, issue 1132): each module's have, and the node's own facts. It reads the key first and writes nothing when the value // is the same, otherwise writes over the revision it read, and creates the key when there is none. A write the bucket // refuses is said and left: the next apply or look writes again, and the report was sent before this ran. -func (q *Queue) writeHaves(ctx context.Context, bus Bus) { +func (q *Session) writeHaves(ctx context.Context, bus Bus) { hb, ok := bus.(HaveBus) if !ok || bus == nil { return @@ -208,9 +222,12 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) { sort.Strings(modules) for _, module := range modules { f := s.modules[module] - h := delivery.Have{V: delivery.Version, Build: f.Build, Phase: delivery.PhaseRunning, - Health: delivery.HealthUnknown, Detail: f.Detail} - if f.Failed { + h := delivery.Have{V: delivery.Version, FP: f.FP, Composed: f.Composed, Build: f.Build, + Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown, Detail: f.Detail} + switch { + case f.Phase != "": + h.Phase = delivery.Phase(f.Phase) + case f.Failed: h.Phase = delivery.PhaseFailed } if got, known := s.health[module]; known { @@ -244,8 +261,10 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) { key, err := delivery.HaveNodeKey(node) if err == nil { var raw []byte - if raw, err = delivery.EncodeHaveNode(*s.node); err == nil { - want := *s.node + facts := *s.node + facts.Composed = s.nodeComposed + if raw, err = delivery.EncodeHaveNode(facts); err == nil { + want := facts q.putHave(write, bucket, key, raw, func(current []byte) bool { c, err := delivery.DecodeHaveNode(current) return err == nil && delivery.SameHaveNode(c, want) @@ -261,7 +280,7 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) { // writeRemoved writes, once, each module the applied declaration no longer holds and this node-engine wrote a have key // for, with the phase removed over the revision it reads (issue 1369). The key is never deleted, and a key that is not // in the bucket is not created. False when the pass should stop. The caller holds the lock of the haves. -func (q *Queue) writeRemoved(ctx context.Context, b HaveBucket, node string) bool { +func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) bool { s := &q.haves gone := make([]string, 0, len(s.removed)) for module := range s.removed { @@ -311,7 +330,7 @@ func (q *Queue) writeRemoved(ctx context.Context, b HaveBucket, node string) boo // putHave reads a key and writes it unless same says it already holds the value. False when the pass should stop: the // bucket failed, the error is said once, and the rest wait for the next pass. -func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool { +func (q *Session) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool { if ctx.Err() != nil { q.sayHave(fmt.Sprintf("the time for writing the have bucket ran out before %s; the next apply or look writes it", key)) return false @@ -324,11 +343,112 @@ func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []b q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err)) return false case same(current): + q.haveWritten() return true } if err := b.Set(ctx, key, value, revision); err != nil { q.sayHave(fmt.Sprintf("could not write %s in the have bucket; the next apply or look writes it again: %v", key, err)) return false } + q.haveWritten() return true } + +// haveWritten tells the caller this node-engine has a have key in the bucket saying what it says now: written by it, or +// already holding the same. It is what proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9): the first +// have under its own build, not a declaration's report. +func (q *Session) haveWritten() { + if q.HaveWritten != nil { + q.HaveWritten() + } +} + +// NoteModule takes how one module's apply came out and writes its have where it differs from the bucket's (novox/hq ADR +// 0539 §9): the node agent's word for a module, with the fingerprint and the digest it runs. It keeps the module's last +// health, which a look changes without an apply. +func (q *Session) NoteModule(ctx context.Context, f ModuleFact) { + q.init() + s := &q.haves + s.mu.Lock() + if s.modules == nil { + s.modules = map[string]ModuleFact{} + } + s.modules[f.Module] = f + delete(s.removed, f.Module) + s.mu.Unlock() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + q.writeHaves(ctx, bus) +} + +// NoteRemoved takes that a module's own resources were removed, and writes its have with the phase removed once (issue +// 1369): the key is never deleted. +func (q *Session) NoteRemoved(ctx context.Context, module, build string) { + q.init() + s := &q.haves + s.mu.Lock() + if s.removed == nil { + s.removed = map[string]string{} + } + if f, ok := s.modules[module]; ok && build == "" { + build = f.Build + } + s.removed[module] = build + delete(s.modules, module) + delete(s.written, module) + s.mu.Unlock() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + q.writeHaves(ctx, bus) +} + +// NoteNode takes the digest of the node's own composition it runs, and writes the node's key where it differs. +func (q *Session) NoteNode(ctx context.Context, composed string) { + q.init() + s := &q.haves + s.mu.Lock() + s.nodeComposed = composed + if s.node == nil { + n := delivery.HaveNode{V: delivery.Version} + s.node = &n + } + s.mu.Unlock() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + q.writeHaves(ctx, bus) +} + +// ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot +// be read answers as none: the agent then compares with the declaration it kept, and applies only what differs. +func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + hb, ok := bus.(HaveBus) + if !ok || bus == nil { + return delivery.Have{}, false + } + key, err := delivery.HaveKey(q.Membership.Node, module) + if err != nil { + return delivery.Have{}, false + } + read, cancel := context.WithTimeout(ctx, q.timeout()) + defer cancel() + bucket, err := hb.HaveBind(read) + if err != nil { + return delivery.Have{}, false + } + value, _, err := bucket.Get(read, key) + if err != nil { + return delivery.Have{}, false + } + h, err := delivery.DecodeHave(value) + if err != nil { + return delivery.Have{}, false + } + return h, true +} diff --git a/internal/link/have_nats_integration_test.go b/internal/link/have_nats_integration_test.go index 1ff1f03..eaa862c 100644 --- a/internal/link/have_nats_integration_test.go +++ b/internal/link/have_nats_integration_test.go @@ -4,7 +4,6 @@ package link import ( "context" - "encoding/json" "sync/atomic" "testing" "time" @@ -32,26 +31,20 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { _ = kv.Purge(node + "." + name) } - m, key := aMember(t) + m, _ := aMember(t) m.Node = node var failB atomic.Bool - var applied atomic.Int32 failB.Store(true) - apply := func(_ context.Context, raw, _ []byte) Report { - applied.Add(1) - var d struct{ Order } - _ = json.Unmarshal(raw, &d) - return Report{Declared: digest(raw), Order: d.Order, Applied: []string{"a.x", "b.x"}, + facts := func() Report { + return Report{Applied: []string{"a.x", "b.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: failB.Load(), Detail: "b.x failed: no space"}}, Health: healthOf(map[string]string{"a": StateHealthy, "b": StateHealthy}), Windows: []Window{{Step: "backup", Holds: []string{"db"}}}} } - q := aQueue(m, apply, &keptInMemory{}) - q.attach(context.Background(), OverNATS{Conn: conn, JS: js}) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - defer func() { stop(); <-worker }() + q := &Session{Membership: m, Timeout: 2 * time.Second} + q.Attach(context.Background(), OverNATS{Conn: conn, JS: js}) + ctx := context.Background() revisions := func() map[string]uint64 { out := map[string]uint64{} @@ -62,14 +55,14 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { } return out } - arrives := func(what string, id string, seq int64) { - want := applied.Load() + 1 - q.Deliver(ordered(t, key, id, 7, seq)) - eventually(t, what, func() bool { return applied.Load() >= want }) + arrives := func(what string) { + if !q.SayFacts(ctx, facts()) { + t.Fatal(what + ": the facts were not taken") + } time.Sleep(300 * time.Millisecond) } - arrives("the first apply", "one", 1) - eventually(t, "the three keys were never written", func() bool { return len(revisions()) == 3 }) + arrives("the first facts") + eventuallyHeld(t, "the three keys were never written", func() bool { return len(revisions()) == 3 }) first := revisions() a, _ := kv.Get(node + ".a") @@ -87,7 +80,7 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { t.Fatalf("the node key holds %+v (%v)", hn, err) } - arrives("the second apply", "two", 2) + arrives("the second facts") time.Sleep(200 * time.Millisecond) for name, rev := range revisions() { if rev != first[name] { @@ -96,8 +89,8 @@ func TestNatsHavesAreWrittenOnceAndOnlyOnChange(t *testing.T) { } failB.Store(false) - arrives("the third apply", "three", 3) - eventually(t, "b was never updated", func() bool { return revisions()["b"] > first["b"] }) + arrives("the third facts") + eventuallyHeld(t, "b was never updated", func() bool { return revisions()["b"] > first["b"] }) if now := revisions(); now["a"] != first["a"] || now["node"] != first["node"] { t.Errorf("a key that did not change moved: %v from %v", now, first) } diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 152e8aa..5402d54 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -126,20 +126,20 @@ func healthOf(states map[string]string) *Health { } func appliedModules() Report { - return Report{Declared: "d1", Applied: []string{"a.x", "b.x"}, + return Report{Applied: []string{"a.x", "b.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}}, Health: healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})} } -func aHaveQueue(l *haveLink) (*Queue, *[]string) { +func aHaveQueue(l *haveLink) (*Session, *[]string) { var said []string var mu sync.Mutex - q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) { + q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) { mu.Lock() said = append(said, s) mu.Unlock() }} - q.attach(context.Background(), l) + q.Attach(context.Background(), l) return q, &said } @@ -148,7 +148,7 @@ func aHaveQueue(l *haveLink) (*Queue, *[]string) { func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) a, b := l.have(t, "n1.a"), l.have(t, "n1.b") if a.Phase != delivery.PhaseRunning || a.Build != "b-1" || a.Health != delivery.HealthOK { t.Errorf("a: %+v", a) @@ -168,7 +168,7 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-0", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.kv["n1.a"].rev != 1 { t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev) } @@ -176,7 +176,7 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev) } before := l.setCount() - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.setCount() != before { t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before) } @@ -186,9 +186,9 @@ func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { func TestAModulesHealthFollowsTheLook(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - rep := Report{Declared: "d1", Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"}, + rep := Report{Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"}, {Module: "c", Build: "b"}, {Module: "d", Build: "b"}}} - q.tell(context.Background(), rep, applied) + q.SayFacts(context.Background(), rep) for _, m := range []string{"a", "b", "c", "d"} { if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown { t.Errorf("%s with no look is %s, want unknown", m, h) @@ -230,7 +230,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { rep.Strays = []Stray{{Kind: "container", Name: "old"}} rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}} rep.Profile = map[string]any{"cores": 8} - q.tell(context.Background(), rep, applied) + q.SayFacts(context.Background(), rep) if len(l.events) < 2 || l.events[0] != "report" { t.Fatalf("the report was not first: %v", l.events) } @@ -255,7 +255,7 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) - if !q.tell(context.Background(), appliedModules(), applied) { + if !q.SayFacts(context.Background(), appliedModules()) { t.Fatal("the report did not go out") } if len(l.said()) != 1 { @@ -267,7 +267,7 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { l.hmu.Lock() l.failing = false l.hmu.Unlock() - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.have(t, "n1.a").Phase != delivery.PhaseRunning { t.Error("the next apply did not write again") } @@ -276,39 +276,19 @@ func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { // A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong. func TestNoBucketAndNoFactsWriteNothing(t *testing.T) { plain := newQuietLink() - q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second} - q.attach(context.Background(), plain) - if !q.tell(context.Background(), appliedModules(), applied) { + q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second} + q.Attach(context.Background(), plain) + if !q.SayFacts(context.Background(), appliedModules()) { t.Fatal("the report did not go out") } l := newHaveLink() q2, _ := aHaveQueue(l) - q2.tell(context.Background(), Report{Refused: "no"}, applied) + q2.SayFacts(context.Background(), Report{Refused: "no"}) if l.setCount() != 0 { t.Errorf("a refusal wrote %d keys", l.setCount()) } } -// 1132/AC5: a reconcile whose report is not news still writes the have keys that differ, so a node-engine that -// restarted writes them again; and writes nothing when they are the same. -func TestASilentReconcileStillWritesTheHaves(t *testing.T) { - l := newHaveLink() - q, _ := aHaveQueue(l) - q.Reconcile = func(context.Context) (Report, bool) { return appliedModules(), true } - q.reconcile(context.Background(), false) - if len(l.said()) != 0 { - t.Fatalf("a reconcile that is not news said %d reports", len(l.said())) - } - if l.have(t, "n1.a").Phase != delivery.PhaseRunning { - t.Error("the reconcile wrote no have") - } - before := l.setCount() - q.reconcile(context.Background(), false) - if l.setCount() != before { - t.Errorf("an unchanged reconcile wrote %d keys", l.setCount()-before) - } -} - // 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that // fails ends the pass, and is said once. func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { @@ -321,7 +301,7 @@ func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { q, said := aHaveQueue(l) q.Timeout = 100 * time.Millisecond start := time.Now() - if !q.tell(context.Background(), appliedModules(), applied) { + if !q.SayFacts(context.Background(), appliedModules()) { t.Fatalf("%s: the report did not go out", name) } if took := time.Since(start); took > time.Second { @@ -334,14 +314,14 @@ func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) if l.binds != 1 || l.setTries != 1 || len(*said) != 1 { t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said) } } func onlyA() Report { - return Report{Declared: "d2", Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}}, + return Report{Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}}, Health: healthOf(map[string]string{"a": StateHealthy})} } @@ -350,9 +330,9 @@ func onlyA() Report { func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) b := l.have(t, "n1.b") if b.Phase != delivery.PhaseRemoved || b.Build != "b-2" { t.Errorf("b: %+v", b) @@ -364,7 +344,7 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev) } before := l.setCount() - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) if l.setCount() != before { t.Errorf("a second apply without b wrote %d keys, removed is written once", l.setCount()-before) } @@ -374,13 +354,13 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { func TestARemovedModuleAddedBackIsWrittenWithItsNewPhase(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) - q.tell(context.Background(), onlyA(), applied) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) + q.SayFacts(context.Background(), onlyA()) + q.SayFacts(context.Background(), appliedModules()) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseFailed || b.Build != "b-2" { t.Errorf("b after it came back: %+v", b) } - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved { t.Errorf("b removed again: %+v", b) } @@ -392,19 +372,19 @@ func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) { l := newHaveLink() l.put(t, "n1.c", delivery.Have{V: delivery.Version, Build: "b-9", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), appliedModules()) + q.SayFacts(context.Background(), onlyA()) if c := l.have(t, "n1.c"); c.Phase != delivery.PhaseRunning || l.kv["n1.c"].rev != 1 { t.Errorf("c, which this node-engine never wrote, changed: %+v at %d", c, l.kv["n1.c"].rev) } // a key that vanished from the bucket is not created by the removal l2 := newHaveLink() q2, _ := aHaveQueue(l2) - q2.tell(context.Background(), appliedModules(), applied) + q2.SayFacts(context.Background(), appliedModules()) l2.hmu.Lock() delete(l2.kv, "n1.b") l2.hmu.Unlock() - q2.tell(context.Background(), onlyA(), applied) + q2.SayFacts(context.Background(), onlyA()) if _, there := l2.kv["n1.b"]; there { t.Error("the removal created a key that was not there") } @@ -414,15 +394,15 @@ func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) { func TestARefusedRemovalIsWrittenByTheNextPass(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) - q.tell(context.Background(), appliedModules(), applied) + q.SayFacts(context.Background(), appliedModules()) l.hmu.Lock() l.failing = true l.hmu.Unlock() - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) l.hmu.Lock() l.failing = false l.hmu.Unlock() - q.tell(context.Background(), onlyA(), applied) + q.SayFacts(context.Background(), onlyA()) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved { t.Errorf("b: %+v", b) } @@ -437,7 +417,7 @@ func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T) q, _ := aHaveQueue(l) // fresh: nothing written by this process r := onlyA() r.Before = []string{"a", "b", "c"} - q.tell(context.Background(), r, applied) + q.SayFacts(context.Background(), r) if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved { t.Errorf("b: %+v", b) } @@ -448,3 +428,73 @@ func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T) t.Errorf("a was written, revision %d", l.kv["n1.a"].rev) } } + +// R6-A AC7: the first have this node-engine writes — or finds already saying what it would write — is told to the +// proof, which proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9); a write the bucket refuses is not. +func TestTheFirstHaveWrittenIsToldToTheProof(t *testing.T) { + l := newHaveLink() + l.failing = true + q, _ := aHaveQueue(l) + var told int + q.HaveWritten = func() { told++ } + q.SayFacts(context.Background(), appliedModules()) + if told != 0 { + t.Fatalf("a refused write proved the build %d time(s)", told) + } + l.hmu.Lock() + l.failing = false + l.hmu.Unlock() + q.SayFacts(context.Background(), appliedModules()) + if told == 0 { + t.Fatal("a have written under this build was not told") + } + + // The bucket already holds what this node-engine would write: a restart with nothing changed still proves it. + same := newHaveLink() + q2, _ := aHaveQueue(same) + q2.SayFacts(context.Background(), appliedModules()) + again, _ := aHaveQueue(same) + told = 0 + again.HaveWritten = func() { told++ } + again.SayFacts(context.Background(), appliedModules()) + if told == 0 { + t.Fatal("a have already saying the same was not told") + } +} + +// R6-A AC6: an applied module's have names its fingerprint and the digest of the composition applied; a health check that +// fails after the module's start window gives health failed, and the have keeps both. +func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(t *testing.T) { + l := newHaveLink() + q, _ := aHaveQueue(l) + fp := "src2:" + strings.Repeat("a", 64) + dig := "sha256:" + strings.Repeat("b", 64) + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig}) + h := l.have(t, "n1.web") + if h.FP != fp || h.Composed != dig || h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthUnknown { + t.Fatalf("the have after the apply: %+v", h) + } + // Starting: it waits. Past its start window the check fails: failed. + q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting})) + if got := l.have(t, "n1.web"); got.Health != delivery.HealthWaits || got.FP != fp || got.Composed != dig { + t.Fatalf("while it starts: %+v", got) + } + q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateUnhealthy})) + if got := l.have(t, "n1.web"); got.Health != delivery.HealthFailed || got.FP != fp || got.Composed != dig { + t.Fatalf("a failing check after the start window: %+v", got) + } + + // A module held for a failed backup says so and what it keeps running; removed is written once, over its key. + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseHeld), Detail: "the repository is locked"}) + if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseHeld || got.Detail == "" { + t.Fatalf("held: %+v", got) + } + q.NoteRemoved(context.Background(), "web", "") + if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved { + t.Fatalf("removed: %+v", got) + } + q.NoteNode(context.Background(), dig) + if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig { + t.Fatalf("the node key: %+v %v", n, err) + } +} diff --git a/internal/link/health_test.go b/internal/link/health_test.go index 3132c4b..db7935a 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -23,14 +23,14 @@ func (l *sayingLink) Health(_ context.Context, node string, body []byte) error { } func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { - q := &Queue{Membership: Membership{Node: "anchor"}} + q := &Session{Membership: Membership{Node: "anchor"}} h := Health{Contract: LivenessContract, At: time.Now().UTC(), Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy, Reason: ReasonRestarting}}} if q.SayHealth(t.Context(), h) { t.Fatal("said with no link open") } l := &sayingLink{quietLink: newQuietLink()} - q.attach(t.Context(), l) + q.Attach(t.Context(), l) if !q.SayHealth(t.Context(), h) || len(l.said) != 1 { t.Fatalf("not said on the open link: %d", len(l.said)) } @@ -42,8 +42,8 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { t.Fatalf("said %+v", got) } // A link that cannot say it is not an error: the next report carries the statement. - q.detach(l) - q.attach(t.Context(), newQuietLink()) + q.Detach(l) + q.Attach(t.Context(), newQuietLink()) if q.SayHealth(t.Context(), h) { t.Fatal("said on a link that cannot") } @@ -77,22 +77,22 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) { // the link was up, because the NATS link could not Ask). With no link open, and with a link that cannot // ask, the check fails and says which. func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) { - q := &Queue{Membership: Membership{Node: "anchor"}} + q := &Session{Membership: Membership{Node: "anchor"}} if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" { t.Fatalf("with no link open: %v", err) } quiet := newQuietLink() - q.attach(t.Context(), quiet) + q.Attach(t.Context(), quiet) if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" { t.Fatalf("on a link that cannot ask: %v", err) } - q.detach(quiet) + q.Detach(quiet) // The link Open returns, with no connection under it: the question must reach its Ask (and fail // there), never stop at the queue's assertion. held := &natsLink{node: "anchor"} - q.attach(t.Context(), held) + q.Attach(t.Context(), held) _, err := q.AskTool(t.Context(), "sensors", "sensors_health") if err == nil { t.Fatal("asked on no connection and was answered") diff --git a/internal/link/hearing.go b/internal/link/hearing.go index e1f4503..0301409 100644 --- a/internal/link/hearing.go +++ b/internal/link/hearing.go @@ -8,9 +8,8 @@ import ( // What a host hears, as the host's own words for it. // -// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and the -// declarations that arrive. The run loop reads its own words for a declaration rather than the -// client library's delivery type, so nothing past this file knows what carried it. +// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and what the controller asks of this +// node. A node hears no declaration: it follows its wants (novox/hq ADR 0539). // // **The host still imports nothing of the mesh's own** (novox/hq ADR 0005). This is its own // interface over its own libraries, and it agrees with the controller only because a conformance @@ -21,12 +20,9 @@ import ( // One interface rather than two, because dialling once is what keeps both halves of a node on the // same connection. type Link interface { - // Bus is what this node says: what it applied, and that it is here. + // Bus is what this node says: its facts, and that it is here. Bus - // Declarations is what the mesh tells this node to be. - Declarations() <-chan Declaration - // Lost says the link ended, and why. // // **Read rather than discovered.** A node that finds out by noticing silence is a node that @@ -38,31 +34,12 @@ type Link interface { Close() } -// Asked is a link that hears the mesh asking this node to say again what it last applied (novox/hq -// to-be 45 §6, the `report` verb). Optional: a link without it is never asked, and the mesh then sends -// the current declaration again instead. +// Asked is a link that hears the mesh asking this node to say its facts again. Optional: a link without it is never +// asked. type Asked interface { AskedToReport() <-chan struct{} } -// Declaration is one thing the mesh told this node to be. -// -// **Handled, once — after the report is published.** A node that dies between applying and -// reporting leaves the declaration with the mesh and applies it again on return, which is safe -// because applying is reconciliation: it converges rather than repeating. -// -// There is one way of being done rather than two. A declaration set aside because a newer arrived -// with it is settled exactly as an applied one is, and the difference between them is a fact the -// *report* carries — a second method here would be a distinction the bus does not make. -type Declaration interface { - // Body is the signed declaration as it arrived, bytes unchanged: a node verifies what it - // received rather than what it re-encoded. - Body() []byte - - // Handled settles it. Called after the report for it has been published, either way. - Handled() error -} - // Open opens this node's link to its mesh. // // Named Open rather than Dial because Dial is this package's raw TLS dial, which the enrolment path diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index ba8d84b..da68fa2 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -11,37 +11,16 @@ import ( "github.com/nats-io/nats.go" ) -// The host's link on the bus being built. -// -// Two things a host does here that it cannot do on the other bus, and one it must not try. -// -// **It declares nothing.** On the bus the mesh has, a host declares its own queue on connecting, -// because a queue that is not there means a node that hears nothing. Here the object it reads -// through is a durable consumer, and a host's account reaches no part of the JetStream API — by -// design, because the controller is the only writer of consumer definitions (design 25 §3). So the -// host **binds** to a consumer the controller made when this node enrolled, and a missing one is -// said as what it is rather than quietly created with whatever configuration this client happens to -// default to. -// -// **It gets order for free, and keeps the drain anyway.** The declaration subject is last-per-subject -// (design 29 §4), so a node that was away receives exactly the current declaration rather than a -// queue of superseded ones, and the stream's sequence orders them definitively — the wire-level -// answer to novox/hq issue 107. What the drain in run.go still answers is the live case: three -// pushes to a *connected* node are three deliveries whatever the stream later retains. +// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration +// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts. // EnrolSubject is where a joining machine asks. One subject for every node, because a machine // enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is // the whole of what its enrolment user may do. const EnrolSubject = "mesh.control.enrol" -// DeclareSubject is where this node's declaration lands. Its own, and no other node's: a host's -// account subscribes exactly this and the subject is the authority on which node a declaration is -// for. -func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" } - -// AskReportSubject is where the mesh asks this node to say again what it last applied (novox/hq to-be 45 -// §6): its own, on core NATS and off any stream. The answer is an ordinary report on its report subject, -// the one thing it already says — nobody's inbox is answered. +// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream. +// The answer is the facts event on its report subject — nobody's inbox is answered. func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" } // natsURL is a bus address as the client wants it. A membership records host and port, because that @@ -58,7 +37,6 @@ func natsURL(address string) string { type natsLink struct { conn *nats.Conn js nats.JetStreamContext - sub *nats.Subscription asking *nats.Subscription asked chan struct{} handing *nats.Subscription @@ -72,9 +50,8 @@ type natsLink struct { moving *nats.Subscription node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). - signer ed25519.PublicKey - arrived chan Declaration - lost chan error + signer ed25519.PublicKey + lost chan error } func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, error) { @@ -123,27 +100,10 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e l := &natsLink{ conn: conn, js: js, node: m.Node, signer: m.Signer, - arrived: make(chan Declaration, drainDepth), - lost: make(chan error, 1), - asked: make(chan struct{}, 1), + lost: make(chan error, 1), + asked: make(chan struct{}, 1), } - // Bound to the consumer the controller made for this node, named after the node because that is - // what the node's own ack grant allows (`$JS.ACK.NODES..>`). - feed := make(chan *nats.Msg, drainDepth) - // The subject as well as the binding: the client checks what is asked for against the - // consumer's own filter, and an empty subject is refused rather than taken to mean "whatever - // that consumer delivers". - sub, err := js.ChanSubscribe(DeclareSubject(m.Node), feed, nats.Bind("NODES", m.Node)) - if err != nil { - conn.Close() - return nil, fmt.Errorf( - "this node cannot read its declarations: %w. The mesh creates that when a node enrols, "+ - "and a host may not create one itself — so this is the mesh's to answer, not this "+ - "machine's", err) - } - l.sub = sub - l.hearAsks() conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { @@ -159,29 +119,6 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e } }) - go func() { - defer close(l.arrived) - for { - select { - case <-ctx.Done(): - return - case msg, ok := <-feed: - if !ok { - select { - case l.lost <- errors.New("the bus stopped delivering"): - default: - } - return - } - select { - case l.arrived <- natsDeclaration{msg}: - case <-ctx.Done(): - return - } - } - } - }() - return l, nil } @@ -299,14 +236,10 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) { } } -func (l *natsLink) Declarations() <-chan Declaration { return l.arrived } -func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } -func (l *natsLink) Lost() <-chan error { return l.lost } +func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } +func (l *natsLink) Lost() <-chan error { return l.lost } func (l *natsLink) Close() { - if l.sub != nil { - _ = l.sub.Unsubscribe() - } if l.asking != nil { _ = l.asking.Unsubscribe() } @@ -375,12 +308,3 @@ var ( _ Asker = (*natsLink)(nil) _ Asked = (*natsLink)(nil) ) - -// natsDeclaration is one declaration off the NODES stream. -type natsDeclaration struct{ msg *nats.Msg } - -func (d natsDeclaration) Body() []byte { return d.msg.Data } - -// Handled acknowledges it. The ack goes to this node's own ack subject, which is the one thing -// besides its reports a node's account may publish. -func (d natsDeclaration) Handled() error { return d.msg.Ack() } diff --git a/internal/link/hearing_nats_test.go b/internal/link/hearing_nats_test.go deleted file mode 100644 index f32bc88..0000000 --- a/internal/link/hearing_nats_test.go +++ /dev/null @@ -1,337 +0,0 @@ -package link - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "fmt" - "os" - "reflect" - "testing" - "time" - - "github.com/nats-io/nats.go" -) - -// The host's link against a real server, because every claim here is about one. -// -// Whether binding to a consumer the host did not create works, whether a declaration on the node's -// own subject arrives, whether acknowledging it removes it from the consumer's pending — none of -// that can be reasoned out, and the first two are the ones that would leave a node silently hearing -// nothing: -// -// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js -// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNats - -func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) { - t.Helper() - url := os.Getenv("MESH_TEST_NATS") - if url == "" { - t.Skip("MESH_TEST_NATS unset") - } - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - // **Ensured and purged, not deleted and recreated.** Delete-then-add looked like a reset and is - // not one: a test that did that inherited the previous test's messages, and the symptom was a - // declaration counted as delivered twice — which reads as a redelivery bug in the code under - // test rather than as a dirty stream. Purge is defined to empty a stream; recreating one is a - // race with the server's own teardown. - for _, want := range []*nats.StreamConfig{ - {Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, MaxMsgsPerSubject: 1}, - {Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, - Retention: nats.WorkQueuePolicy}, - } { - if _, err := js.StreamInfo(want.Name); err != nil { - if _, err := js.AddStream(want); err != nil { - t.Fatal(err) - } - } - if err := js.PurgeStream(want.Name); err != nil { - t.Fatal(err) - } - } - return conn, js -} - -// theMeshMakes is the consumer the controller creates when a node enrols. Made here by the test -// because the host may not: its account reaches no part of the JetStream API, which is the whole -// reason this binds rather than subscribes. -// -// Removed afterwards, and each test names its own node: two tests sharing a consumer name share its -// delivery count and its pending list, and the first thing that goes wrong reads as a fault in the -// host rather than in the test beside it. -func theMeshMakes(t *testing.T, js nats.JetStreamContext, node string) { - t.Helper() - t.Cleanup(func() { _ = js.DeleteConsumer("NODES", node) }) - if _, err := js.AddConsumer("NODES", &nats.ConsumerConfig{ - Durable: node, - FilterSubject: DeclareSubject(node), - AckPolicy: nats.AckExplicitPolicy, - AckWait: 300 * time.Second, - DeliverSubject: "_DELIVER." + node, - }); err != nil { - t.Fatal(err) - } -} - -func signedBy(t *testing.T, key ed25519.PrivateKey, declaration []byte) []byte { - t.Helper() - body, err := json.Marshal(Signed{ - Declaration: declaration, Signature: ed25519.Sign(key, declaration), - }) - if err != nil { - t.Fatal(err) - } - return body -} - -// A declaration on this node's own subject reaches the host, is applied, and acknowledging it -// empties the consumer — which is what tells the mesh the node has it. -func TestNatsADeclarationReachesTheHostAndIsSettled(t *testing.T) { - conn, js := aBus(t) - const node = "settling" - theMeshMakes(t, js, node) - - public, private, _ := ed25519.GenerateKey(nil) - m := Membership{Node: node, Signer: public} - - // Dialled directly rather than through Open: the test server has no TLS, and what is being - // checked is the subscription and the settling, not the pin — which PinnedConfig owns and its - // own tests cover. - l := &natsLink{conn: conn, js: js, node: node, - arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)} - feed := make(chan *nats.Msg, drainDepth) - sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node)) - if err != nil { - t.Fatalf("the host could not bind to the consumer the mesh made for it: %v", err) - } - defer func() { _ = sub.Unsubscribe() }() - go func() { - for msg := range feed { - l.arrived <- natsDeclaration{msg} - } - }() - - if _, err := js.Publish(DeclareSubject(node), - signedBy(t, private, []byte(`{"declared":"d1"}`))); err != nil { - t.Fatal(err) - } - - select { - case d := <-l.Declarations(): - report := handleBody(context.Background(), m, d.Body(), - func(context.Context, []byte, []byte) Report { - return Report{Applied: []string{"store"}} - }) - if report.Refused != "" { - t.Fatalf("a declaration the mesh signed was refused: %s", report.Refused) - } - if err := d.Handled(); err != nil { - t.Fatalf("the node could not acknowledge its own declaration: %v", err) - } - case <-time.After(8 * time.Second): - t.Fatal("no declaration reached the host") - } - - // **Nothing pending is the property**; a delivery count is not. Delivery is at-least-once by - // design, so pinning "delivered exactly once" would be asserting something the mesh does not - // rely on. What matters is that the acknowledgement landed, so the mesh can tell the node has - // it — and that no redelivery was needed to get there, which is what would say the node was - // too slow to answer for its own ack wait. - deadline := time.Now().Add(5 * time.Second) - var last string - for time.Now().Before(deadline) { - info, err := js.ConsumerInfo("NODES", node) - switch { - case err != nil: - last = err.Error() - case info.NumAckPending == 0 && info.NumRedelivered == 0: - return - default: - last = fmt.Sprintf("pending %d, redelivered %d", info.NumAckPending, info.NumRedelivered) - } - time.Sleep(20 * time.Millisecond) - } - t.Fatalf("the declaration was not settled, so the mesh cannot tell the node has it: %s", last) -} - -// **A node that was away gets exactly the current declaration and nothing older.** Three pushed -// while nothing is listening leave one on the stream, and it is the newest — the wire-level answer -// to novox/hq issue 107, and the half of the drain that stops being the host's problem. -func TestNatsANodeThatWasAwayGetsOnlyTheNewest(t *testing.T) { - _, js := aBus(t) - const node = "returning" - _, private, _ := ed25519.GenerateKey(nil) - - for _, id := range []string{"d1", "d2", "d3"} { - if _, err := js.Publish(DeclareSubject(node), - signedBy(t, private, []byte(`{"declared":"`+id+`"}`))); err != nil { - t.Fatal(err) - } - } - info, err := js.StreamInfo("NODES") - if err != nil { - t.Fatal(err) - } - if info.State.Msgs != 1 { - t.Fatalf("%d declarations survived for one node; a node that was away would apply a backlog "+ - "of things nobody wants any more", info.State.Msgs) - } - - theMeshMakes(t, js, node) - feed := make(chan *nats.Msg, drainDepth) - sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node)) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - - select { - case msg := <-feed: - if want := declaredIn(signedBy(t, private, []byte(`{"declared":"d3"}`))); declaredIn(msg.Data) != want { - t.Fatalf("the node was given %q rather than the newest", msg.Data) - } - case <-time.After(8 * time.Second): - t.Fatal("the node that was away was given nothing") - } -} - -// A report goes through the stream: it must survive the controller's store restarting. -func TestNatsAReportIsKept(t *testing.T) { - conn, js := aBus(t) - bus := OverNATS{Conn: conn, JS: js} - ctx := context.Background() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1"}) - if err := bus.Report(ctx, "anchor", body); err != nil { - t.Fatal(err) - } - - info, err := js.StreamInfo("CONTROL") - if err != nil { - t.Fatal(err) - } - if info.State.Msgs != 1 { - t.Fatalf("%d messages were kept; a report must be, once", info.State.Msgs) - } -} - -// **One apply queue, against a real bus** (novox/hq to-be 45 §6, R2). A declaration is being applied -// when two more are pushed and the five-minute reconcile comes due: the worker applies the one in -// hand, then only the newest — the reconcile is that apply — and the mesh's stream holds one account -// per apply, each naming the order it applied, numbered in the order they were made. Every -// declaration delivered is settled. -func TestNatsTheQueueAppliesTheNewestOnceAndReportsInOrder(t *testing.T) { - conn, js := aBus(t) - const node = "queueing" - theMeshMakes(t, js, node) - public, private, _ := ed25519.GenerateKey(nil) - m := Membership{Node: node, Signer: public} - - ctx, stop := context.WithCancel(context.Background()) - defer stop() - l := &natsLink{conn: conn, js: js, node: node, - arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)} - feed := make(chan *nats.Msg, drainDepth) - sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node)) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - go func() { - for msg := range feed { - l.arrived <- natsDeclaration{msg} - } - }() - reports, err := js.SubscribeSync(ReportSubject(node), nats.BindStream("CONTROL")) - if err != nil { - t.Fatal(err) - } - defer func() { _ = reports.Unsubscribe() }() - - a := &applies{} - inFirst, release := make(chan struct{}, 1), make(chan struct{}) - reconciled := 0 - q := &Queue{Membership: m, Timeout: 5 * time.Second, Unsaid: &keptInMemory{}, - Apply: func(ctx context.Context, raw, sig []byte) Report { - r := a.apply(ctx, raw, sig) - if r.Sequence == 1 { - inFirst <- struct{}{} - <-release - } - return r - }, - Reconcile: func(context.Context) (Report, bool) { reconciled++; return Report{}, false }, - } - worker := runWorker(ctx, q) - go func() { _ = serve(ctx, l, m, q, nil, 5*time.Second) }() - - push := func(id string, sequence int64) { - inner, _ := json.Marshal(map[string]any{"declaration": 1, "epoch": 7, "sequence": sequence, - "resources": []any{map[string]any{"id": id}}}) - if _, err := js.Publish(DeclareSubject(node), signedBy(t, private, inner)); err != nil { - t.Fatal(err) - } - } - push("one", 1) - select { - case <-inFirst: - case <-time.After(8 * time.Second): - t.Fatal("the first declaration never reached the worker") - } - push("two", 2) - push("three", 3) - q.ReconcileDue() - time.Sleep(2 * drainWindow) // the link gathers both and hands them to the queue - close(release) - - var heard []Report - for len(accounts(heard)) < 2 { - msg, err := reports.NextMsg(8 * time.Second) - if err != nil { - t.Fatalf("the mesh heard %+v and then nothing: %v", heard, err) - } - var r Report - if err := json.Unmarshal(msg.Data, &r); err != nil { - t.Fatal(err) - } - heard = append(heard, r) - _ = msg.Ack() - } - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"one", "three"}) { - t.Fatalf("applied %v; the one in hand and then only the newest were wanted", got) - } - if reconciled != 0 { - t.Fatal("the reconcile applied beside the delivery it was due with") - } - accounted := accounts(heard) - if accounted[0].Sequence != 1 || accounted[1].Sequence != 3 || accounted[1].Epoch != 7 { - t.Fatalf("the accounts do not name what was applied: %+v", accounted) - } - for i := 1; i < len(heard); i++ { - if heard[i].ReportSequence <= heard[i-1].ReportSequence { - t.Fatalf("the reports left out of the order they were made: %+v", heard) - } - } - deadline := time.Now().Add(5 * time.Second) - for { - info, err := js.ConsumerInfo("NODES", node) - if err == nil && info.NumAckPending == 0 { - break - } - if time.Now().After(deadline) { - t.Fatalf("a delivered declaration was left unsettled: %+v %v", info, err) - } - time.Sleep(20 * time.Millisecond) - } -} diff --git a/internal/link/held_note_test.go b/internal/link/held_note_test.go deleted file mode 100644 index 43b9953..0000000 --- a/internal/link/held_note_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package link - -import ( - "strings" - "testing" -) - -// The count that did not add up was the only symptom sixteen held resources had, and reading it meant -// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did -// says what it did not, too. - -func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) { - got := heldNote([]Held{ - {ID: "ca", Module: "route-proxy", Kind: "directory"}, - {ID: "certs", Module: "route-proxy", Kind: "directory"}, - {ID: "server", Module: "route-proxy", Kind: "container"}, - {ID: "mail", Module: "mailu", Kind: "container"}, - }) - if !strings.Contains(got, "4 held") { - t.Fatalf("the count of what was held is not in the line: %q", got) - } - // The module is the thing an operator can act on: `take` takes a module. - if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") { - t.Fatalf("the line does not break the holds down by module: %q", got) - } - // Ordered, so two machines holding the same things read the same and a diff of two reports is - // about what changed. - if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") { - t.Fatalf("modules are not in a stated order: %q", got) - } - // It says why, because "held" alone reads as a failure and this is correct behaviour. - if !strings.Contains(got, "taken") { - t.Fatalf("the line does not say a hold ends when the module is taken: %q", got) - } -} - -func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) { - // A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of - // them is how a line stops being read. - if got := heldNote(nil); got != "" { - t.Fatalf("an apply with no holds added %q to its line", got) - } - if got := heldNote([]Held{}); got != "" { - t.Fatalf("an apply with no holds added %q to its line", got) - } -} diff --git a/internal/link/messages.go b/internal/link/messages.go index 406c75d..3aaba65 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -18,22 +18,6 @@ const ( KeyReport = "report" ) -// Signed is a declaration and the signature over it. -// -// novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is -// verified by its signature, every time**. The two are different questions — a node connects to -// the broker and takes instruction from the control plane behind it, and pinning only the first -// would make the second transitive. -// -// The signature is over Declaration exactly as it arrived, bytes unchanged. Re-encoding before -// verifying would mean checking a signature over something other than what was sent, and any -// difference in key order or spacing would break it — so the raw message is what is signed and -// what is checked. -type Signed struct { - Declaration []byte `json:"declaration"` - Signature []byte `json:"signature"` -} - // Report is what a node says after applying, and it is a statement rather than a write. // // A node states; the context that owns the data writes (novox/hq ADR 0006). The difference is the @@ -51,14 +35,6 @@ type Report struct { // Refused is set when the declaration was rejected whole rather than applied in part. Refused string `json:"refused,omitempty"` - // Superseded names the newer declaration this one was set aside for, unapplied. - // - // A machine asked to be five successive things becomes the last one (novox/hq issue 031): - // when several declarations are waiting, the host applies the newest and acknowledges the - // rest without applying them. Each of those is still reported, because silence reads as a - // machine that ignored an instruction and "applied" would be a lie — this is the third word. - Superseded string `json:"superseded,omitempty"` - // Carried are the machine's ports held by what this host raised from its own bundle. // // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node @@ -70,62 +46,13 @@ type Report struct { // machine saying what is true of it, not asking for anything. Carried []int `json:"carried,omitempty"` - // Declared is the digest of the declaration this report is about — sha256 of the exact bytes - // the mesh sent, which the mesh recorded when it sent them. - // - // **Which declaration, not when.** The mesh compared its send time to this report's arrival - // to decide whether a machine had caught up, and lost the race it invited: an apply started - // under the previous declaration finishes after the next one is sent, its report lands newer - // than the send, and the machine reads as caught up with words it has not read yet. Clocks - // cannot answer "which"; the digest is the answer itself. - Declared string `json:"declared,omitempty"` - - // Order is where the declaration this report is about stands — its `epoch` and `sequence`, as - // the declaration carried them — so the mesh orders reports by what they are about rather than - // by when they arrived (novox/hq to-be 45 §6). On the wire as two top-level keys beside - // `declared`; absent for a declaration that claimed no order, and for a report about none. + // Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand + // among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine, + // the sequence one higher for every facts event, so the controller refuses older facts by this order. On the + // wire as three top-level keys. + Writer string `json:"writer,omitempty"` Order - // ReportSequence is this node-engine's own order among everything it reports: one higher for - // every report it makes, kept on disk so it goes on increasing across restarts and self-updates - // (to-be 45 §6). With the declaration's order it lets the mesh refuse an older report about the - // same declaration — a reconcile's account overtaking the apply that followed it (novox/hq issue - // 267) is then refused by number rather than set aside by digest. A report said again because - // it never reached the mesh (issue 264) carries the number it was made with. - // - // Zero claims no order: every report an older host makes, and a one-shot report a command makes - // (a rekey). **Its presence also says this host reads a declaration's `epoch`**, which is how the - // mesh knows it may send one: an older host decodes a declaration strictly and refuses a key it - // does not know. - ReportSequence int64 `json:"report_sequence,omitempty"` - - // OlderThan is set on a report refusing a declaration older than one this node has applied: the - // order of the newer declaration it holds (to-be 45 §6, rule 2). The refused declaration is the - // report's own Declared and Order, and Refused says it in words. Applying it would make the - // machine into something a controller that lost its lease said, after one that holds it. - OlderThan *Order `json:"older_than,omitempty"` - - // RefusedOlder is how many declarations this node-engine has refused as older, ever — kept on - // disk with the report sequence. On every report, so a count missed with a lost refusal is still - // read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts. - RefusedOlder int64 `json:"refused_older,omitempty"` - - // ReadsGeneration says this node-engine reads a declaration's assignment generation and its - // put-back mark (novox/hq issue 234), on every report it makes: the controller sends them only to a - // machine that said so, because an older node-engine decodes strictly and refuses a key it does not - // know, whole. A node-engine rolled back says so no longer. - ReadsGeneration bool `json:"reads_generation,omitempty"` - - // ReadsModules says this node-engine reads a resource's `module` and a declaration's `builds` (novox/hq ADR 0429 - // §4, issue 1132), on every report it makes, beside ReadsGeneration: the controller sends them only to a machine - // that said so, for the same reason. - ReadsModules bool `json:"reads_modules,omitempty"` - - // OlderGeneration is set on a report refusing a declaration composed from an older assignment - // generation than this machine applied (novox/hq issue 234). The refused declaration is the report's - // own Declared and Order, so the controller can name who sent it from its record of sends. - OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"` - // Held is what this adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` @@ -647,52 +574,14 @@ type Rollback struct { At time.Time `json:"at"` } -// Order is where a declaration stands among everything the mesh has sent this node (novox/hq to-be -// 45 §6): the controller's lease **epoch** it was sent under, and its **sequence** — one higher for -// every send to this node (04-ISSUES/107). The declaration carries both inside what is signed, as -// top-level `epoch` and `sequence` beside `declaration`; a report carries back the pair of the -// declaration it is about. -// -// Zero in either is "no order claimed", never "first": every declaration an older controller sent, -// and the bundle genesis applies. +// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its +// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one +// with nothing kept on disk. type Order struct { Epoch int64 `json:"epoch,omitempty"` Sequence int64 `json:"sequence,omitempty"` } -// Older says whether a declaration of this order is older than one of order `than`, which this node -// has applied — the one rule by which a node-engine refuses a declaration (to-be 45 §6: "older epoch; -// same epoch, lower sequence"). -// -// **Only when both claim an epoch.** A declaration with none is an older controller's — or a -// controller rolled back to a build from before the lease — and one kept with none is what this host -// held before any controller had a lease. Neither has an order to compare, and refusing on a guess -// would strand the machine the moment the controller is older than the host: without an epoch, today's -// behaviour stands. A higher epoch is a new lease holder and is never older, whatever its sequence. -func (o Order) Older(than Order) bool { - if o.Epoch <= 0 || than.Epoch <= 0 { - return false - } - if o.Epoch != than.Epoch { - return o.Epoch < than.Epoch - } - return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence -} - -// Supersedes says whether a declaration of order `o`, arriving after one of order `before`, takes its -// place among what is waiting to be applied. By epoch when both claim one and they differ, then by -// sequence when both claim one, and by arrival when they do not — what the drain had to go on before -// declarations said where they stand (04-ISSUES/107). -func (o Order) Supersedes(before Order) bool { - if o.Epoch > 0 && before.Epoch > 0 && o.Epoch != before.Epoch { - return o.Epoch > before.Epoch - } - if o.Sequence > 0 && before.Sequence > 0 { - return o.Sequence >= before.Sequence - } - return true -} - // Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a // Stringer promoted onto every report would print each one as its order alone. func (o Order) Words() string { @@ -842,18 +731,16 @@ type FoundFirewall struct { RetiredBy string `json:"retired_by,omitempty"` } -// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that -// generation, and the highest this machine applied (novox/hq issue 234). -type GenerationRefusal struct { - Generation int64 `json:"generation"` - Applied int64 `json:"applied"` -} - -// ModuleFact is how one module's part of an apply came out: the build the node was sent for it, and whether every -// resource of it applied. Detail names the failing resource and its error (issue 1132). +// ModuleFact is how one module's part of an apply came out (novox/hq ADR 0539 §9): the fingerprint FP the module runs and +// the digest Composed of the composition applied for it, beside the build the node was sent, the Phase it is in (empty +// is running, or failed when Failed), and whether every resource of it applied. Detail names the failing resource and its +// error (issue 1132). type ModuleFact struct { - Module string - Build string - Failed bool - Detail string + Module string + Build string + FP string + Composed string + Phase string + Failed bool + Detail string } diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index d2cb235..b27f39b 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -1,115 +1,10 @@ package link import ( - "context" - "crypto/ed25519" "encoding/json" "testing" ) -// verified runs what Run does to a delivery body, without a broker: unmarshal, check the -// signature, and only then apply. Isolating it keeps this test about the check rather than about -// the bus, which is tested against a real one in the lab. -func verified(t *testing.T, signer ed25519.PublicKey, body []byte) (Report, bool) { - t.Helper() - applied := false - report := handleBody(context.Background(), Membership{Node: "anchor", Signer: signer}, body, - func(context.Context, []byte, []byte) Report { - applied = true - return Report{Applied: []string{"something"}} - }) - return report, applied -} - -func signedBody(t *testing.T, private ed25519.PrivateKey, declaration string) []byte { - t.Helper() - raw, err := json.Marshal(Signed{ - Declaration: []byte(declaration), - Signature: ed25519.Sign(private, []byte(declaration)), - }) - if err != nil { - t.Fatal(err) - } - return raw -} - -func TestTheMeshsOwnDeclarationIsApplied(t *testing.T) { - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - report, applied := verified(t, public, signedBody(t, private, `{"declaration":1}`)) - if !applied { - t.Fatalf("a declaration the mesh signed was not applied: %s", report.Refused) - } -} - -// Defends novox/hq ADR 0002: everything reaching a node arrives over the broker — and therefore -// ADR 0004's consequence, that the broker is not trusted to say who is speaking. -// -// A transport nobody authenticates per-message would let whatever holds the connection attribute -// a declaration to any node it liked. -func TestAForgedDeclarationIsNeverApplied(t *testing.T) { - // The check that stands between "the mesh changes this machine" and "anybody does". The host - // applies whatever the link delivers, so a forged declaration is the whole machine. - public, _, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - _, other, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - - report, applied := verified(t, public, signedBody(t, other, `{"declaration":1}`)) - if applied { - t.Fatal("a declaration signed by another key was applied") - } - if report.Refused != ErrForged.Error() { - t.Errorf("refused, but not as a forgery: %q", report.Refused) - } -} - -func TestATamperedDeclarationIsNeverApplied(t *testing.T) { - // A broker that changed the declaration in flight, keeping the signature. This is what makes - // pinning the transport insufficient on its own. - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - raw, err := json.Marshal(Signed{ - Declaration: []byte(`{"declaration":1,"resources":["something else entirely"]}`), - Signature: ed25519.Sign(private, []byte(`{"declaration":1}`)), - }) - if err != nil { - t.Fatal(err) - } - - report, applied := verified(t, public, raw) - if applied { - t.Fatal("a declaration altered after signing was applied") - } - if report.Refused != ErrForged.Error() { - t.Errorf("refused, but not as a forgery: %q", report.Refused) - } -} - -func TestAMalformedMessageIsToldApartFromAForgery(t *testing.T) { - // novox/hq ADR 0004 requires these to be distinguishable: one means somebody is trying, the - // other means something is broken, and they need different responses from a person. - public, _, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - report, applied := verified(t, public, []byte("this is not a message")) - if applied { - t.Fatal("something unparseable was applied") - } - if report.Refused == ErrForged.Error() { - t.Error("a malformed message was reported as a forgery; those must be distinguishable") - } -} - func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { // The contract with the control plane, which defines these separately. A matching test lives // there; rename a field on either side and both fail. @@ -117,7 +12,9 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { value any expect []string }{ - {Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}}, + // novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence. + {Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}}, + []string{"node", "writer", "epoch", "sequence"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, // novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what diff --git a/internal/link/movesecret.go b/internal/link/movesecret.go index fdfee75..95463bb 100644 --- a/internal/link/movesecret.go +++ b/internal/link/movesecret.go @@ -173,7 +173,7 @@ const VaultWithin = 15 * time.Second // AskVault asks one of the vault's verbs in this node's engine's own name over the link open now, and answers its // result, or its refusal as an error. -func (q *Queue) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) { +func (q *Session) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) { q.init() q.mu.Lock() bus := q.bus diff --git a/internal/link/movesecret_test.go b/internal/link/movesecret_test.go index 5cb244a..41ae611 100644 --- a/internal/link/movesecret_test.go +++ b/internal/link/movesecret_test.go @@ -112,8 +112,8 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) { } defer vault.Unsubscribe() l := &natsLink{conn: conn, js: js, node: m.Node, signer: m.Signer} - q := &Queue{Membership: m} - q.attach(t.Context(), l) + q := &Session{Membership: m} + q.Attach(t.Context(), l) l.HearMoveSecrets(func(s MoveSecret) (MovedSecret, error) { raw, err := q.AskVault(t.Context(), "move-in", map[string]string{"module": s.Module}) if err != nil || !strings.Contains(string(raw), `"call":"c1"`) { @@ -135,7 +135,7 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) { if err := json.Unmarshal(reply.Data, &said); err != nil || said.Secret != "own/"+m.Node+"/postgres/superuser-password" { t.Fatalf("answered %s, %v", reply.Data, err) } - if _, err := (&Queue{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) { + if _, err := (&Session{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) { t.Fatalf("a queue with no link asked the vault: %v", err) } } diff --git a/internal/link/newest_test.go b/internal/link/newest_test.go deleted file mode 100644 index 5eb12a5..0000000 --- a/internal/link/newest_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package link - -import ( - "sync" - "testing" - "time" -) - -type said struct { - body []byte - - mu sync.Mutex - handled bool -} - -func (s *said) Body() []byte { return s.body } -func (s *said) Handled() error { - s.mu.Lock() - defer s.mu.Unlock() - s.handled = true - return nil -} -func (s *said) wasHandled() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.handled -} - -func arriving(bodies ...string) chan Declaration { - ch := make(chan Declaration, 8) - for _, b := range bodies { - ch <- &said{body: []byte(b)} - } - return ch -} - -// newest is what the queue's worker would take from what the link gathered behind `first`. -func newest(arriving <-chan Declaration, first Declaration, window time.Duration) (Declaration, []Declaration) { - return pick(gather(arriving, first, window)) -} - -// A machine asked to be five things becomes the last one: what is already waiting supersedes what -// arrived first, and everything set aside is named so it can be reported. -func TestWhatIsAlreadyWaitingSupersedesWhatArrivedFirst(t *testing.T) { - waiting := arriving("two", "three", "four") - apply, superseded := newest(waiting, &said{body: []byte("one")}, 50*time.Millisecond) - if string(apply.Body()) != "four" { - t.Fatalf("applied %q, not the newest", apply.Body()) - } - if len(superseded) != 3 || string(superseded[0].Body()) != "one" || - string(superseded[2].Body()) != "three" { - t.Fatalf("set aside %d: %v", len(superseded), superseded) - } -} - -// One declaration with nothing behind it is applied as it always was, after the window. -func TestALoneDeclarationIsAppliedAfterTheWindow(t *testing.T) { - began := time.Now() - apply, superseded := newest(arriving(), &said{body: []byte("only")}, 30*time.Millisecond) - if string(apply.Body()) != "only" || len(superseded) != 0 { - t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded)) - } - if time.Since(began) < 30*time.Millisecond { - t.Fatal("did not wait the window for a straggler") - } -} - -// A straggler within the window is taken; one after it is the next push. -func TestAStragglerWithinTheWindowIsTaken(t *testing.T) { - waiting := arriving() - go func() { - time.Sleep(20 * time.Millisecond) - waiting <- &said{body: []byte("late")} - }() - apply, superseded := newest(waiting, &said{body: []byte("first")}, 100*time.Millisecond) - if string(apply.Body()) != "late" || len(superseded) != 1 { - t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded)) - } -} diff --git a/internal/link/nothing_alive_test.go b/internal/link/nothing_alive_test.go index 300bea5..3945bb0 100644 --- a/internal/link/nothing_alive_test.go +++ b/internal/link/nothing_alive_test.go @@ -35,7 +35,7 @@ func (l *listeningForAlive) count() int { func TestTheLinkLoopPublishesNoAlive(t *testing.T) { m, _ := aMember(t) l := &listeningForAlive{quietLink: newQuietLink()} - q := aQueue(m, nil, &keptInMemory{}) + q := &Session{Membership: m, Timeout: time.Second} ctx, stop := context.WithCancel(context.Background()) done := make(chan error, 1) diff --git a/internal/link/order_test.go b/internal/link/order_test.go deleted file mode 100644 index 529d06c..0000000 --- a/internal/link/order_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package link - -import ( - "encoding/json" - "testing" - "time" -) - -// The drain picked the last to arrive. A backlog longer than the batch, or a broker that split a -// burst, delivered a superseded declaration last (novox/hq 04-ISSUES/107). - -func sequenced(t *testing.T, n int64) *said { - t.Helper() - inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{}, "sequence": n}) - if err != nil { - t.Fatal(err) - } - body, err := json.Marshal(Signed{Declaration: inner, Signature: []byte("s")}) - if err != nil { - t.Fatal(err) - } - return &said{body: body} -} - -func TestTheDrainKeepsTheHighestSequenceNotTheLastToArrive(t *testing.T) { - waiting := make(chan Declaration, 8) - waiting <- sequenced(t, 9) - waiting <- sequenced(t, 4) // arrived last, composed earlier - latest, aside := newest(waiting, sequenced(t, 8), 30*time.Millisecond) - if got := orderOf(latest.Body()).Sequence; got != 9 { - t.Fatalf("the drain kept sequence %d, and 9 was waiting", got) - } - if len(aside) != 2 { - t.Fatalf("%d set aside, wanted 2 (the 8 and the late 4)", len(aside)) - } -} - -func TestWithoutSequencesTheLastToArriveStillWins(t *testing.T) { - // The behaviour this had before, kept for a controller that sends no order. - apply, aside := newest(arriving("two", "three"), &said{body: []byte("one")}, 30*time.Millisecond) - if string(apply.Body()) != "three" || len(aside) != 2 { - t.Fatalf("applied %q with %d set aside", apply.Body(), len(aside)) - } -} - -func TestAnUnreadableBodyClaimsNoOrder(t *testing.T) { - if got := orderOf([]byte("not json")); got != (Order{}) { - t.Fatalf("garbage claimed %+v", got) - } -} diff --git a/internal/link/queue.go b/internal/link/queue.go deleted file mode 100644 index ddc2ea5..0000000 --- a/internal/link/queue.go +++ /dev/null @@ -1,647 +0,0 @@ -package link - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/health" -) - -// Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1). -// -// **One worker applies; everything else asks it to.** A declaration delivered over the link and the -// five-minute reconcile are two reasons to *enqueue* the same act, never two paths that apply. Before -// this they were two, each taking an apply lock, and every order between them was a fault met on the -// live mesh: the reconcile read what was kept before a delivery and applied it over the newer one -// (novox/hq issues 257, 261); the reconcile went first and its report about the older declaration -// reached the mesh after the delivery's (267). Each fix closed one order and left the class open. -// -// The queue holds at most one pending request, coalesced: whatever has been delivered and not yet -// taken, and whether a reconcile is due. When the worker starts it takes the **newest declaration held -// at that moment**, sets the rest aside — each reported as set aside, then settled unapplied — applies -// it once, and makes **one report**, naming the declaration's order and its own report sequence. A -// reconcile due while a delivery waits is that delivery's apply: applying the newest thing the mesh -// said is what reconciling is. Only with nothing delivered does a reconcile apply what was kept, read -// once its turn has come. -// -// **Reports leave in the order they are made**, because the worker makes and says them, one at a time. -// A report made while the link is down waits: an apply's in the unsaid store (issue 264), a -// reconcile's in one slot replaced by anything newer, and both are said, in that order, when the next -// link opens — before anything newly delivered is applied. -type Queue struct { - // Membership is whose declarations these are: their signer, and the node the reports are from. - Membership Membership - // Apply applies a declaration the mesh signed. - Apply Applier - // Reconcile holds the machine to what it was last told, read once it is this act's turn. False - // when there is nothing to hold it to, or the reconcile could not run; it says why itself. - Reconcile Reconcile - // News is whether a reconcile's report is worth saying unasked (novox/hq ADR 0100, ADR 0140). Nil - // is never: a reconcile is otherwise silent. - News func(Report) bool - // Heard is told every account of the machine the mesh has taken — an apply's or a reconcile's, - // not a refusal or a declaration set aside, which say nothing about the machine. Nil is allowed. - Heard func(Report) - // Unsaid keeps an apply's report until the mesh has taken it (issue 264). Nil keeps nothing. - Unsaid Unsaid - // Numbers keeps the report sequence and the refusals counted, across restarts and self-updates. - // Nil counts in memory, which is only right where nothing outlives the process — a test. - Numbers Numbers - // HandOver judges and records a directory used as found handed to the mesh by the operator, asked by - // the controller over the link (novox/hq issue 356). Nil answers every such ask with a refusal. - HandOver HandOverFunc - // SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the - // controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal. - SetuidSearch SetuidSearchFunc - // JudgeDirectories judges this node's declared directories read-only, asked by the controller over the - // link (novox/hq issue 358). Nil answers every such ask with a refusal. - JudgeDirectories JudgeDirectoriesFunc - // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved - // warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal. - AccessAct AccessActFunc - // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of - // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. - MoveSecret MoveSecretFunc - Say Announce - Timeout time.Duration - - once sync.Once - wake chan struct{} - - mu sync.Mutex - idle *sync.Cond - // delivering is a delivered declaration in hand: its report is owed on the link it came over, so - // that link is not let go until it is said (detach). A reconcile in hand holds no link — its report - // waits for the next one if this one goes. - delivering bool - waiting []Declaration // delivered and not yet taken, in the order they arrived - due bool // a reconcile asked for - // asked is the mesh asking this node to say what it last applied (the `report` verb, to-be 45 §6): - // the reconcile it enqueues says its report whether or not it is news. - asked bool - bus Bus // the link open now; nil while there is none - unasked *Report // a reconcile's report not yet said, made while there was no link - - // saying is held while a report is made and said, so the order they leave in is the order they - // are made in — whoever says them, the worker or a link opening. - saying sync.Mutex - counted bool - numbers numbered - - // haves is what this node-engine writes beside its reports (novox/hq ADR 0429 §4, issue 1132). - haves haveState -} - -// Reconcile is the reconcile's act, run by the queue's worker. See Queue.Reconcile. -type Reconcile func(ctx context.Context) (Report, bool) - -// Numbers is where the queue keeps what it counts, so a successor goes on from where it stopped. -type Numbers interface { - // Read is what was kept: zero for a node that never reported, an error when it cannot be read — - // never zero for "could not tell". - Read() (reportSequence, refusedOlder int64, err error) - // Save keeps them, on disk when it returns. - Save(reportSequence, refusedOlder int64) error -} - -type numbered struct{ sequence, refused int64 } - -func (q *Queue) init() { - q.once.Do(func() { - q.wake = make(chan struct{}, 1) - q.idle = sync.NewCond(&q.mu) - if q.Say == nil { - q.Say = func(string) {} - } - }) -} - -// Deliver enqueues what the link delivered, in the order it arrived. It never waits for an apply. -func (q *Queue) Deliver(arrived ...Declaration) { - q.init() - q.mu.Lock() - q.waiting = append(q.waiting, arrived...) - q.mu.Unlock() - q.signal() -} - -// ReconcileDue enqueues a reconcile. One asked for while another is waiting is the same one. -func (q *Queue) ReconcileDue() { - q.init() - q.mu.Lock() - q.due = true - q.mu.Unlock() - q.signal() -} - -// ReportAsked is the mesh asking this node to say again what it last applied (novox/hq to-be 45 §6, the -// `report` verb its healer H1 asks when a send went unreported). **An enqueue, like everything else**: a -// reconcile, whose report — the account of the declaration this node keeps, the last it applied — is -// said whether or not it is news. A delivery waiting meanwhile is applied instead and reported, which -// answers the question better. Asked again before the worker takes it is the same question. -func (q *Queue) ReportAsked() { - q.init() - q.mu.Lock() - q.due, q.asked = true, true - q.mu.Unlock() - q.signal() -} - -func (q *Queue) signal() { - select { - case q.wake <- struct{}{}: - default: - // One is already waiting to be read, and the worker takes everything pending when it does. - } -} - -// Run is the worker. It returns once the context ends and the act in hand is finished and said: -// asked to stop — the host standing aside for its successor — nothing further is applied. -func (q *Queue) Run(ctx context.Context) { - q.init() - for { - for q.step(ctx) { - } - select { - case <-ctx.Done(): - return - case <-q.wake: - } - } -} - -// step takes what is pending and does it, once. False when there was nothing to do, or the queue -// was asked to stop. -func (q *Queue) step(ctx context.Context) bool { - q.mu.Lock() - if ctx.Err() != nil || (len(q.waiting) == 0 && !q.due) { - q.mu.Unlock() - return false - } - batch, due, asked := q.waiting, q.due, q.asked - q.waiting, q.due, q.asked = nil, false, false - q.delivering = len(batch) > 0 - q.mu.Unlock() - defer func() { - q.mu.Lock() - q.delivering = false - q.idle.Broadcast() - q.mu.Unlock() - }() - - if len(batch) == 0 { - q.reconcile(ctx, asked) - return true - } - - latest, aside := pick(batch) - for _, old := range aside { - if d := declaredIn(old.Body()); d != "" && d == declaredIn(latest.Body()) { - // The same declaration again — delivered twice while the worker was busy. Nothing is set - // aside: it is about to be applied. - _ = old.Handled() - continue - } - q.Say(fmt.Sprintf("set aside declaration %s (%s): a newer one arrived with it", - short(declaredIn(old.Body())), orderOf(old.Body()).Words())) - q.tell(ctx, Report{Declared: declaredIn(old.Body()), Order: orderOf(old.Body()), - Superseded: declaredIn(latest.Body())}, setAside, old) - } - - report := handleBody(q.withFiles(ctx), q.Membership, latest.Body(), q.Apply) - switch { - case report.OlderThan != nil: - // **Refused, counted and said** (rule 2): one line here, the count on this and every later - // report, and the report itself naming what was refused and what is held. - total := q.countRefusal() - q.Say(fmt.Sprintf("refused declaration %s (%s): this node applied %s, from a newer lease "+ - "holder — %d refused as older so far", short(report.Declared), report.Order.Words(), - report.OlderThan.Words(), total)) - case report.OlderGeneration != nil: - // Said here, and the report names it, so the controller raises it naming the sender (novox/hq - // issue 234). Not counted with the lease's refusals: those are S13's, by epoch. - q.Say(fmt.Sprintf("refused declaration %s (%s): it was composed from assignment generation %d, and "+ - "this node applied generation %d", short(report.Declared), report.Order.Words(), - report.OlderGeneration.Generation, report.OlderGeneration.Applied)) - case report.Refused != "": - q.Say("refused a declaration: " + report.Refused) - case len(report.Failed) > 0: - q.Say(fmt.Sprintf("applied %d and failed: %v%s", - len(report.Applied), report.Failed, heldNote(report.Held))) - default: - q.Say(fmt.Sprintf("applied %d resource(s)%s", len(report.Applied), heldNote(report.Held))) - } - if due && report.Refused != "" { - // Nothing was applied, so the reconcile this stood in for has not happened. It runs next. - q.mu.Lock() - q.due = true - q.mu.Unlock() - } - // A refusal as older is not kept to be said again: what the mesh waits for from this node is the - // account of the last apply, and that is still what is kept. - kind := applied - if report.OlderThan != nil || report.OlderGeneration != nil { - kind = refusedOlder - } - q.tell(ctx, report, kind, latest) - return true -} - -func (q *Queue) reconcile(ctx context.Context, asked bool) { - if q.Reconcile == nil { - if asked { - q.Say("asked to say what this node last applied, and it holds nothing to reconcile with") - } - return - } - report, ok := q.Reconcile(q.withFiles(ctx)) - if !ok { - if asked { - q.Say("asked to say what this node last applied, and the reconcile could not say it") - } - return - } - if !asked && (q.News == nil || !q.News(report)) { - // Not worth a report, but a reconcile found how each module is now: the have keys say it (issue 1132), so a - // node-engine that restarted writes them again without waiting for a change. - if q.noteApply(report) { - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - q.writeHaves(ctx, bus) - } - return - } - if asked { - q.Say(fmt.Sprintf("asked by the mesh, saying again what this node last applied: declaration %s (%s)", - short(report.Declared), report.Order.Words())) - } - q.tell(ctx, report, unasked) -} - -// What a report is, for what tell does with it besides saying it. -type kindOf int - -const ( - // applied is an apply's account: kept until said (issue 264), and fresher than any reconcile's - // report still waiting. - applied kindOf = iota - // setAside is a declaration a newer one took the place of, unapplied. - setAside - // refusedOlder is a declaration refused as older than what was applied. Not kept: what the mesh - // waits for from this node is still the account of the last apply. - refusedOlder - // unasked is a reconcile's report. Not said — no link, or the bus refused it — the newest waits - // for the next link, and anything the worker says before then replaces it. - unasked -) - -// tell makes a report this node's — its node, its report sequence, the refusals counted — and says it -// on the link open now. Kept first, when it is an apply's, so a report lost between here and the bus -// is said by whichever host links next. Each declaration it settles is settled after it is said, -// either way. True when the broker took it. -func (q *Queue) tell(ctx context.Context, r Report, kind kindOf, settle ...Declaration) bool { - q.saying.Lock() - defer q.saying.Unlock() - - r = q.stamp(r) - keep := kind == applied - if keep { - // What this says supersedes any reconcile's report still waiting to be said. - q.unasked = nil - if q.Unsaid != nil { - if err := q.Unsaid.Keep(r); err != nil { - q.Say("cannot keep this apply's report until it is said: " + err.Error()) - } - } - } - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - - published := false - if bus != nil { - // **Said even when the link is being let go** (issue 264): not cancelled with it, still - // bounded by the timeout, and the link is not closed until this returns (detach). - published = publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) - } - if !published && kind == unasked { - q.unasked = &r - } - if published { - // After the report, which is sent first and unchanged; a write the bucket refuses never undoes it (issue 1132). - if q.noteApply(r) { - q.writeHaves(ctx, bus) - } - if keep && q.Unsaid != nil { - if err := q.Unsaid.Said(r.Declared); err != nil { - q.Say("said this apply's report, and cannot forget it: " + err.Error()) - } - } - if q.Heard != nil && (kind == applied || kind == unasked) && r.Refused == "" { - q.Heard(r) - } - } - // Settled after the report is published. A node that dies between applying and reporting - // leaves the declaration with the mesh and applies it again on return, which is safe because - // applying is reconciliation — it converges rather than repeating. - for _, d := range settle { - _ = d.Handled() - } - return published -} - -// stamp gives a report its node, the next report sequence and the refusals counted. Called holding -// saying. -func (q *Queue) stamp(r Report) Report { - q.load() - q.numbers.sequence++ - q.keepNumbers() - r.Node = q.Membership.Node - r.ReportSequence = q.numbers.sequence - r.RefusedOlder = q.numbers.refused - // Every report of a node-engine that orders its reports says it reads a generation (novox/hq issue - // 234): this build does, and the controller sends one only to a machine that said so. - r.ReadsGeneration = true - // And the module of each resource and the build of each module (novox/hq ADR 0429 §4, issue 1132), the same way. - r.ReadsModules = true - return r -} - -func (q *Queue) countRefusal() int64 { - q.saying.Lock() - defer q.saying.Unlock() - q.load() - q.numbers.refused++ - q.keepNumbers() - return q.numbers.refused -} - -// load reads what was counted, once. **Unreadable is said, and the sequence goes on from above -// anything it can have reached** — the time in milliseconds — rather than from one, which would make -// every report this host says next older than what it already said, and refused. -func (q *Queue) load() { - if q.counted { - return - } - q.counted = true - if q.Numbers == nil { - return - } - sequence, refused, err := q.Numbers.Read() - if err != nil { - floor := time.Now().UnixMilli() - q.Say(fmt.Sprintf("cannot read this node's report sequence: %v — going on from %d, above any "+ - "it can have reached, and counting refusals again from none", err, floor)) - q.numbers = numbered{sequence: floor} - return - } - q.numbers = numbered{sequence: sequence, refused: refused} -} - -func (q *Queue) keepNumbers() { - if q.Numbers == nil { - return - } - if err := q.Numbers.Save(q.numbers.sequence, q.numbers.refused); err != nil { - // Said, and not fatal: the report is still worth saying. What is lost is that a successor - // could number one again. - q.Say("cannot keep this node's report sequence: " + err.Error()) - } -} - -// attach is a link opening. **What the last apply did, if the mesh never heard it**, is said first, -// exactly as it was made; then a reconcile's report that waited; and only then may the worker say -// anything on it — so a report said again can never land after, and read as newer than, a later one. -func (q *Queue) attach(ctx context.Context, bus Bus) { - q.init() - q.saying.Lock() - defer q.saying.Unlock() - reporting := context.WithoutCancel(ctx) - q.load() - - said := true - if q.Unsaid != nil { - switch kept, ok, err := q.Unsaid.Pending(); { - case err != nil: - q.Say("cannot read the report this node kept unsaid: " + err.Error()) - case ok: - q.Say("saying again what the last apply did: its report never reached the mesh") - if said = publishReport(reporting, bus, q.Membership, kept, q.Say, q.Timeout); said { - if err := q.Unsaid.Said(kept.Declared); err != nil { - q.Say("said the kept report, and cannot forget it: " + err.Error()) - } - if q.Heard != nil && kept.Refused == "" { - q.Heard(kept) - } - } - } - } - if said && q.unasked != nil { - if publishReport(reporting, bus, q.Membership, *q.unasked, q.Say, q.Timeout) { - if q.Heard != nil { - q.Heard(*q.unasked) - } - q.unasked = nil - } - } - - q.mu.Lock() - q.bus = bus - q.mu.Unlock() -} - -// withFiles is ctx carrying the link now open, so an apply can fetch a declaration's files given by digest (novox/hq ADR -// 0486, issue 1695). A queue with no link open carries none. -func (q *Queue) withFiles(ctx context.Context) context.Context { - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return ctx - } - return WithFiles(ctx, bus) -} - -// detach is the link closing: it waits for a delivery in hand to be applied and said — an apply that -// stood aside for its successor is reported on this link before it is let go (issue 264) — and then -// nothing more is said on it. A reconcile in hand is not waited for: a link that was dropped or roused -// is opened again at once, and the reconcile's report, if it is news, waits for it. -func (q *Queue) detach(bus Bus) { - q.init() - q.mu.Lock() - for q.delivering { - q.idle.Wait() - } - q.mu.Unlock() - - q.saying.Lock() - defer q.saying.Unlock() - q.mu.Lock() - if q.bus == bus { - q.bus = nil - } - q.mu.Unlock() -} - -// pick is the newest of what was delivered and the rest, in arrival order, to set aside: by order -// when the declarations claim one, by arrival when they do not (Order.Supersedes). -func pick(batch []Declaration) (Declaration, []Declaration) { - latest := batch[0] - var aside []Declaration - for _, next := range batch[1:] { - if orderOf(next.Body()).Supersedes(orderOf(latest.Body())) { - aside = append(aside, latest) - latest = next - continue - } - aside = append(aside, next) - } - return latest, aside -} - -// orderOf is the order a signed declaration claims, or none when it claims none or cannot be read. -// Read from the envelope alone; the signature is verified when the winner is applied, and a forged -// message that lied about its order would only set aside real ones — which are reported as set aside, -// and the next push sends the current one again. -func orderOf(body []byte) Order { - var signed Signed - if err := json.Unmarshal(body, &signed); err != nil { - return Order{} - } - var o Order - if err := json.Unmarshal(signed.Declaration, &o); err != nil || o.Epoch < 0 || o.Sequence < 0 { - return Order{} - } - return o -} - -// declaredIn names a signed declaration as the mesh does — the digest of the declaration's bytes, the -// same the apply's report carries — for a report about one that was not applied. Empty if the message -// is not one: a forged or garbled message is refused when its turn comes; here it is only named. -func declaredIn(body []byte) string { - var signed Signed - if err := json.Unmarshal(body, &signed); err != nil || len(signed.Declaration) == 0 { - return "" - } - sum := sha256.Sum256(signed.Declaration) - return hex.EncodeToString(sum[:]) -} - -// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the -// machine's networking its engine judges). -func (q *Queue) Linked() bool { - q.init() - q.mu.Lock() - defer q.mu.Unlock() - return q.bus != nil -} - -// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether -// the bus took it. **Not through the worker**: a statement is a word about the machine as it is, not an -// account of an apply, and it must not wait behind one — a container crash-looping while a long apply -// runs is said while it runs. False with no link open, or one that cannot say it; the caller says it -// again on its next look, and the next report carries it whatever happens. -func (q *Queue) SayHealth(ctx context.Context, h Health) bool { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - sayer, ok := bus.(HealthBus) - if bus == nil || !ok { - return false - } - body, err := json.Marshal(HealthSaid{Node: q.Membership.Node, Health: h}) - if err != nil { - return false - } - saying, cancel := context.WithTimeout(ctx, q.timeout()) - defer cancel() - if err := sayer.Health(saying, q.Membership.Node, body); err != nil { - q.Say("could not say this machine's health: " + err.Error()) - return false - } - // A look that changed a module's health changes its have (issue 1132). - if q.noteHealth(h) { - q.writeHaves(ctx, bus) - } - return true -} - -func (q *Queue) timeout() time.Duration { - if q.Timeout <= 0 { - return 10 * time.Second - } - return q.Timeout -} - -// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR -// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open -// or nothing answered — said as the check's failure, never as healthy. -func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return health.Answer{}, errors.New("no link to the bus is open") - } - asker, ok := bus.(ToolBus) - if !ok { - return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool") - } - reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}")) - if err != nil { - return health.Answer{}, err - } - return ReadToolAnswer(reply) -} - -// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over -// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for. -func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error) { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) - } - asker, ok := bus.(ToolBus) - if !ok { - return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked) - } - return asker.Ask(ctx, CLISubject(q.Membership.Node), body) -} - -// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any -// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272). -var ErrNotAsked = errors.New("not asked") - -// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call — -// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the -// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the -// SDK cannot check is unhealthy too, so a malformed one never excuses anything. -func ReadToolAnswer(reply []byte) (health.Answer, error) { - var env struct { - Result json.RawMessage `json:"result"` - Error string `json:"error"` - } - if err := json.Unmarshal(reply, &env); err != nil { - return health.Answer{}, fmt.Errorf("its answer is not one: %w", err) - } - if env.Error != "" { - return health.Unhealthy(env.Error), nil - } - a, err := health.Read(env.Result) - if err != nil { - return health.Unhealthy("it answered something other than healthy or not"), nil - } - return a, nil -} diff --git a/internal/link/queue_test.go b/internal/link/queue_test.go deleted file mode 100644 index 3e0e7b2..0000000 --- a/internal/link/queue_test.go +++ /dev/null @@ -1,588 +0,0 @@ -package link - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "errors" - "reflect" - "strings" - "sync" - "testing" - "time" -) - -// The node-engine's one apply queue (novox/hq to-be 45 §6): a delivery and the reconcile are two -// reasons to enqueue the same act, the newest declaration is applied once, and the reports leave in -// the order they are made. These are the races of issues 257, 261, 264 and 267 replayed against it. - -func aQueue(m Membership, apply Applier, kept Unsaid) *Queue { - return &Queue{Membership: m, Apply: apply, Unsaid: kept, Timeout: time.Second} -} - -// runWorker runs the queue's worker until ctx ends; the channel closes when it has. -func runWorker(ctx context.Context, q *Queue) <-chan struct{} { - done := make(chan struct{}) - go func() { - defer close(done) - q.Run(ctx) - }() - return done -} - -// ordered is a signed declaration claiming an order, with a resource id so two are distinct bytes. -func ordered(t *testing.T, key ed25519.PrivateKey, id string, epoch, sequence int64) *said { - t.Helper() - inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{map[string]any{"id": id}}, - "epoch": epoch, "sequence": sequence}) - if err != nil { - t.Fatal(err) - } - return &said{body: signedBy(t, key, inner)} -} - -// applies records what the worker applied, by the id of the declaration's first resource, and -// reports it as applied — with the order the declaration carried, as the host's own apply does. -type applies struct { - mu sync.Mutex - seen []string -} - -func (a *applies) apply(_ context.Context, raw, _ []byte) Report { - var d struct { - Resources []struct { - ID string `json:"id"` - } `json:"resources"` - Order - } - _ = json.Unmarshal(raw, &d) - id := "" - if len(d.Resources) > 0 { - id = d.Resources[0].ID - } - a.mu.Lock() - a.seen = append(a.seen, id) - a.mu.Unlock() - return Report{Declared: digest(raw), Order: d.Order, Applied: []string{id}} -} - -func (a *applies) all() []string { - a.mu.Lock() - defer a.mu.Unlock() - return append([]string(nil), a.seen...) -} - -func digest(raw []byte) string { - b, _ := json.Marshal(Signed{Declaration: raw}) - return declaredIn(b) -} - -// eventually waits for a condition the worker reaches on its own time. -func eventually(t *testing.T, what string, ok func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for !ok() { - if time.Now().After(deadline) { - t.Fatal(what) - } - time.Sleep(5 * time.Millisecond) - } -} - -// accounts is the reports the mesh heard that are an apply's account, not a set-aside's. -func accounts(reports []Report) []Report { - var out []Report - for _, r := range reports { - if r.Superseded == "" { - out = append(out, r) - } - } - return out -} - -// **R2: a reconcile due during a delivery** (issues 257, 261, 267). The reconcile is asked for while a -// delivery waits: the worker applies the delivery once — applying the newest thing the mesh said is -// what reconciling is — and makes one report, naming the newest sequence. The reconcile never applies -// what was kept before. -func TestAReconcileDueDuringADeliveryIsThatDeliverysApply(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - reconciled := 0 - q := aQueue(m, a.apply, &keptInMemory{}) - q.Reconcile = func(context.Context) (Report, bool) { - reconciled++ - return Report{Declared: "the one kept before", Applied: []string{"old"}}, true - } - q.News = func(Report) bool { return true } - q.attach(context.Background(), l) - - q.Deliver(ordered(t, key, "new", 7, 12)) - q.ReconcileDue() - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"new"}) { - t.Fatalf("applied %v; one apply of the newest was wanted", got) - } - if reconciled != 0 { - t.Fatalf("the reconcile applied what was kept %d time(s) beside the delivery", reconciled) - } - r := l.said()[0] - if r.Sequence != 12 || r.Epoch != 7 || r.ReportSequence != 1 || r.Node != m.Node { - t.Fatalf("the report does not name the order it applied: %+v", r) - } -} - -// **The other order** (issue 267): the reconcile is running when a delivery arrives. Its report, about -// the declaration kept then, leaves first; the delivery's leaves after it, with a higher report -// sequence. The mesh's latest word is the newest declaration, whichever arrived first. -func TestAReconcileRunningWhenADeliveryArrivesIsSaidBeforeIt(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - inReconcile, release := make(chan struct{}), make(chan struct{}) - q.Reconcile = func(context.Context) (Report, bool) { - close(inReconcile) - <-release - return Report{Declared: "d1", Order: Order{Epoch: 7, Sequence: 11}, Applied: []string{"a"}, - Outward: []string{"eth0"}}, true - } - q.News = func(Report) bool { return true } - q.attach(context.Background(), l) - - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.ReconcileDue() - <-inReconcile - q.Deliver(ordered(t, key, "b", 7, 12)) - close(release) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 2 }) - stop() - <-worker - - reports := l.said() - if reports[0].Declared != "d1" || reports[1].Sequence != 12 { - t.Fatalf("the reports left out of the order they were made: %+v", reports) - } - if reports[0].ReportSequence >= reports[1].ReportSequence { - t.Fatalf("the later report carries no higher report sequence: %d then %d", - reports[0].ReportSequence, reports[1].ReportSequence) - } -} - -// **Two deliveries in quick succession**: only the newest is applied, and there is one account of an -// apply — the first is reported as set aside, naming the one that took its place. -func TestTwoDeliveriesInQuickSuccessionApplyOnlyTheNewest(t *testing.T) { - m, key := aMember(t) - first, second := ordered(t, key, "first", 7, 12), ordered(t, key, "second", 7, 13) - l := newQuietLink(first, second) - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - - ctx, stop := context.WithCancel(context.Background()) - defer stop() - worker := runWorker(ctx, q) - go func() { _ = serve(ctx, l, m, q, nil, time.Second) }() - eventually(t, "nothing was reported", func() bool { return len(accounts(l.said())) == 1 }) - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"second"}) { - t.Fatalf("applied %v; only the newest was wanted", got) - } - reports := l.said() - if len(reports) != 2 || reports[0].Superseded != declaredIn(second.body) || - reports[0].Declared != declaredIn(first.body) || reports[0].Sequence != 12 { - t.Fatalf("the first was not reported as set aside for the second: %+v", reports) - } - if !first.wasHandled() || !second.wasHandled() { - t.Fatal("a declaration was left unsettled") - } -} - -// Deliveries that arrive while the worker is applying are coalesced: when it is free it takes the -// newest held at that moment — by order, not by arrival — and applies it once. -func TestDeliveriesWhileTheWorkerIsBusyAreOneApplyOfTheNewest(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - inApply, release := make(chan struct{}, 1), make(chan struct{}) - q := aQueue(m, func(ctx context.Context, raw, sig []byte) Report { - r := a.apply(ctx, raw, sig) - if r.Sequence == 1 { - inApply <- struct{}{} - <-release - } - return r - }, &keptInMemory{}) - q.attach(context.Background(), l) - - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.Deliver(ordered(t, key, "one", 7, 1)) - <-inApply - q.Deliver(ordered(t, key, "three", 7, 3)) - q.Deliver(ordered(t, key, "two", 7, 2)) // arrived last, composed earlier - q.ReconcileDue() // and the five-minute timer fired too - close(release) - eventually(t, "the newest was never reported", func() bool { return len(accounts(l.said())) == 2 }) - stop() - <-worker - - if got := a.all(); !reflect.DeepEqual(got, []string{"one", "three"}) { - t.Fatalf("applied %v; the one in hand and then only the newest were wanted", got) - } - last := accounts(l.said())[1] - if last.Sequence != 3 { - t.Fatalf("the last account names sequence %d, not the newest", last.Sequence) - } -} - -// **An older epoch is refused, counted and said** (rule 2): the refusal's report names the refused -// declaration and what this node holds, carries the count, and does not replace the kept account of -// the last apply — which is what the mesh is waiting for from this node. -func TestADeclarationFromAnOlderEpochIsRefusedCountedAndSaid(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - kept := &keptInMemory{} - lastApply := Report{Declared: "d-applied", Order: Order{Epoch: 57, Sequence: 3}, Applied: []string{"a"}} - _ = kept.Keep(lastApply) - stale := ordered(t, key, "stale", 41, 12) - q := aQueue(m, func(_ context.Context, raw, _ []byte) Report { - // What the host's apply answers for a declaration older than the one it kept. - return Report{Declared: digest(raw), Order: Order{Epoch: 41, Sequence: 12}, - OlderThan: &Order{Epoch: 57, Sequence: 3}, Refused: "older than what this node applied"} - }, kept) - var lines saidSoFar - q.Say = lines.say - l.refusing = true // nothing is said on linking: the kept account stays kept - q.attach(context.Background(), l) - l.mu.Lock() - l.refusing = false - l.mu.Unlock() - - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.Deliver(stale) - eventually(t, "the refusal was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - - r := l.said()[0] - if r.OlderThan == nil || *r.OlderThan != (Order{Epoch: 57, Sequence: 3}) || r.Epoch != 41 || - r.Declared != declaredIn(stale.body) || r.RefusedOlder != 1 { - t.Fatalf("the refusal does not name what was refused, what is held, and the count: %+v", r) - } - if !strings.Contains(lines.all(), "1 refused as older so far") { - t.Fatalf("the refusal was not said in the log:\n%s", lines.all()) - } - if still, ok, _ := kept.Pending(); !ok || still.Declared != "d-applied" { - t.Fatalf("the refusal replaced the kept account of the last apply: %+v", still) - } - if !stale.wasHandled() { - t.Fatal("the refused declaration was not settled: the mesh would deliver it again") - } -} - -// **Restart with an unsaid report**: the next host says it first, exactly as it was made, and numbers -// what it says next above it — the report sequence goes on across the restart rather than from one. -func TestAfterARestartTheUnsaidReportIsSaidFirstAndTheNumbersGoOn(t *testing.T) { - m, key := aMember(t) - kept := &keptInMemory{} - numbers := &numbersInMemory{} - - // The first host applies; its report never reaches the mesh, and it is gone. - first := newQuietLink() - first.refusing = true - q1 := aQueue(m, (&applies{}).apply, kept) - q1.Numbers = numbers - q1.attach(context.Background(), first) - ctx1, stop1 := context.WithCancel(context.Background()) - w1 := runWorker(ctx1, q1) - q1.Deliver(ordered(t, key, "a", 7, 12)) - eventually(t, "the first host never kept its report", func() bool { _, ok, _ := kept.Pending(); return ok }) - stop1() - <-w1 - lost, _, _ := kept.Pending() - - // The next host links, and is then delivered something new. - next := newQuietLink() - q2 := aQueue(m, (&applies{}).apply, kept) - q2.Numbers = numbers - q2.attach(context.Background(), next) - ctx2, stop2 := context.WithCancel(context.Background()) - w2 := runWorker(ctx2, q2) - q2.Deliver(ordered(t, key, "b", 7, 13)) - eventually(t, "the next host's apply was never reported", func() bool { return len(next.said()) == 2 }) - stop2() - <-w2 - - reports := next.said() - if !reflect.DeepEqual(reports[0], lost) { - t.Fatalf("the unsaid report was not said first, as it was made: %+v", reports[0]) - } - if reports[1].Sequence != 13 || reports[1].ReportSequence <= lost.ReportSequence { - t.Fatalf("the next host numbered from one again: %d after %d", - reports[1].ReportSequence, lost.ReportSequence) - } -} - -// A report sequence that cannot be read is said, and numbering goes on above anything it can have -// reached — never from one, which would make every report older than those already said. -func TestUnreadableNumbersAreSaidAndNumberingGoesOnAboveThem(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - q := aQueue(m, nil, nil) - q.Numbers = &numbersInMemory{broken: errors.New("unreadable")} - var lines saidSoFar - q.Say = lines.say - q.News = func(Report) bool { return true } - q.Reconcile = func(context.Context) (Report, bool) { return Report{Declared: "d"}, true } - before := time.Now().UnixMilli() - q.attach(context.Background(), l) - - q.ReconcileDue() - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "nothing was reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - if got := l.said()[0].ReportSequence; got < before { - t.Fatalf("numbering went on from %d, not above anything it can have reached", got) - } - if !strings.Contains(lines.all(), "cannot read this node's report sequence") { - t.Fatalf("the unreadable sequence was not said:\n%s", lines.all()) - } -} - -// A reconcile's report made with no link waits for the next one; an apply made before then replaces -// it, because the apply's account is fresher. -func TestAReconcileReportMadeOfflineWaitsAndIsReplacedByAnApply(t *testing.T) { - m, key := aMember(t) - q := aQueue(m, (&applies{}).apply, &keptInMemory{}) - q.News = func(Report) bool { return true } - q.Reconcile = func(context.Context) (Report, bool) { - return Report{Declared: "d1", Outward: []string{"eth0"}}, true - } - - ctx, stop := context.WithCancel(context.Background()) - defer stop() - runWorker(ctx, q) - q.ReconcileDue() - eventually(t, "the reconcile's report was not held", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked != nil - }) - l := newQuietLink() - q.attach(context.Background(), l) - if reports := l.said(); len(reports) != 1 || reports[0].Declared != "d1" { - t.Fatalf("the reconcile's report was not said when the link opened: %+v", reports) - } - - // Offline again: a reconcile's report, then an apply of a delivery still in hand. - q.detach(l) - q.ReconcileDue() - eventually(t, "the second reconcile's report was not held", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked != nil - }) - q.Deliver(ordered(t, key, "x", 7, 2)) - eventually(t, "the apply did not replace the reconcile's report", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked == nil - }) -} - -// The contract with the controller, on the wire: the keys a declaration's order is read from, and the -// keys a report carries back. A rename here must break this test before it breaks a machine. -func TestTheOrderOnTheWire(t *testing.T) { - body, err := json.Marshal(Report{Node: "n", Declared: "d", Order: Order{Epoch: 57, Sequence: 3}, - ReportSequence: 9, OlderThan: &Order{Epoch: 58, Sequence: 1}, RefusedOlder: 2}) - if err != nil { - t.Fatal(err) - } - var wire map[string]any - _ = json.Unmarshal(body, &wire) - want := map[string]any{"node": "n", "declared": "d", "epoch": 57.0, "sequence": 3.0, - "report_sequence": 9.0, "older_than": map[string]any{"epoch": 58.0, "sequence": 1.0}, - "refused_older": 2.0} - if !reflect.DeepEqual(wire, want) { - t.Fatalf("a report's order on the wire is\n%s\nnot the contract", body) - } - - // An older host's report, and an older controller's declaration, claim no order. - if body, _ := json.Marshal(Report{Node: "n", Declared: "d"}); strings.Contains(string(body), "sequence") || - strings.Contains(string(body), "epoch") { - t.Fatalf("a report about a declaration with no order claims one: %s", body) - } - _, key := aMember(t) - if got := orderOf(signedBy(t, key, []byte(`{"declaration":1,"epoch":57,"sequence":3}`))); got != (Order{Epoch: 57, Sequence: 3}) { - t.Fatalf("a declaration's order was read as %+v", got) - } -} - -func TestOlder(t *testing.T) { - for _, c := range []struct { - in, held Order - older bool - }{ - {Order{41, 12}, Order{57, 3}, true}, // an older lease holder, whatever its sequence - {Order{57, 2}, Order{57, 3}, true}, // same holder, lower sequence - {Order{57, 3}, Order{57, 3}, false}, // the same declaration again: reconciling - {Order{58, 1}, Order{57, 3}, false}, // a new lease holder - {Order{0, 1}, Order{57, 3}, false}, // an older controller, or one rolled back: today's behaviour - {Order{41, 1}, Order{0, 9}, false}, // nothing held claims an epoch - {Order{57, 0}, Order{57, 3}, false}, // no sequence to compare - {Order{0, 0}, Order{0, 0}, false}, // neither claims anything - {Order{-1, 0}, Order{57, 3}, false}, // never a claim - {Order{57, 4}, Order{57, 3}, false}, // newer - {Order{56, 99}, Order{57, 1}, true}, // a higher sequence is no excuse for an older epoch - {Order{57, 1}, Order{56, 99}, false}, // nor a lower one a reason to refuse a newer epoch - {Order{57, 3}, Order{57, 0}, false}, // held with no sequence - {Order{100, 0}, Order{57, 0}, false}, // newer epoch, no sequences - {Order{10, 0}, Order{57, 0}, true}, // older epoch, no sequences - {Order{57, 2}, Order{0, 0}, false}, // nothing held at all - {Order{0, 2}, Order{0, 3}, false}, // sequence alone does not refuse on the link (no epoch) - {Order{57, 2}, Order{57, -1}, false}, // a held sequence below zero is not one - {Order{57, -1}, Order{57, 2}, false}, // nor an arriving one - {Order{-5, -5}, Order{-1, -1}, false}, // nothing below zero is an order - } { - if got := c.in.Older(c.held); got != c.older { - t.Errorf("%+v older than %+v = %v, want %v", c.in, c.held, got, c.older) - } - } -} - -func TestSupersedes(t *testing.T) { - for _, c := range []struct { - next, before Order - want bool - }{ - {Order{58, 1}, Order{57, 9}, true}, // a new lease holder - {Order{57, 9}, Order{58, 1}, false}, // the stale one arriving late - {Order{57, 4}, Order{57, 3}, true}, - {Order{57, 2}, Order{57, 3}, false}, // arrived last, composed earlier - {Order{0, 4}, Order{0, 3}, true}, - {Order{0, 2}, Order{0, 3}, false}, - {Order{}, Order{0, 3}, true}, // no order: by arrival - {Order{0, 3}, Order{}, true}, - } { - if got := c.next.Supersedes(c.before); got != c.want { - t.Errorf("%+v supersedes %+v = %v, want %v", c.next, c.before, got, c.want) - } - } -} - -// numbersInMemory is Numbers without a disk, surviving a "restart" by being shared. -type numbersInMemory struct { - mu sync.Mutex - sequence, refused int64 - broken error -} - -func (n *numbersInMemory) Read() (int64, int64, error) { - n.mu.Lock() - defer n.mu.Unlock() - return n.sequence, n.refused, n.broken -} - -func (n *numbersInMemory) Save(sequence, refused int64) error { - n.mu.Lock() - defer n.mu.Unlock() - n.sequence, n.refused = sequence, refused - return nil -} - -// A link dropped or roused while a reconcile is in hand is let go at once and opened again: the -// reconcile holds no link, and its report, if it is news, waits for the next one. Only a delivery in -// hand keeps its link until its report is said. -func TestALinkIsNotHeldForAReconcileInHand(t *testing.T) { - m, _ := aMember(t) - inReconcile, release := make(chan struct{}), make(chan struct{}) - q := aQueue(m, nil, nil) - q.News = func(Report) bool { return true } - q.Reconcile = func(context.Context) (Report, bool) { - close(inReconcile) - <-release - return Report{Declared: "d1", Outward: []string{"eth0"}}, true - } - l := newQuietLink() - q.attach(context.Background(), l) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - q.ReconcileDue() - <-inReconcile - - detached := make(chan struct{}) - go func() { q.detach(l); close(detached) }() - select { - case <-detached: - case <-time.After(2 * time.Second): - t.Fatal("the link was held while a reconcile ran") - } - close(release) - eventually(t, "the reconcile's report was not held for the next link", func() bool { - q.saying.Lock() - defer q.saying.Unlock() - return q.unasked != nil - }) - stop() - <-worker - if len(l.said()) != 0 { - t.Fatalf("a report was said on a link let go: %+v", l.said()) - } -} - -// **Every report says this node-engine reads a declaration's generation** (novox/hq issue 234): the -// controller sends one only to a machine that said so, because an older node-engine refuses the key. -func TestEveryReportSaysItReadsAGeneration(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - q.attach(context.Background(), l) - q.Deliver(ordered(t, key, "new", 7, 12)) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - raw, err := json.Marshal(l.said()[0]) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(raw), `"reads_generation":true`) { - t.Fatalf("the report does not say it reads a generation: %s", raw) - } -} - -// 1132/AC2: every report says this node-engine reads a resource's module and a declaration's builds, beside the -// generation: the controller sends them only to a machine that said so, because an older node-engine refuses the keys. -func TestEveryReportSaysItReadsModules(t *testing.T) { - m, key := aMember(t) - l := newQuietLink() - a := &applies{} - q := aQueue(m, a.apply, &keptInMemory{}) - q.attach(context.Background(), l) - q.Deliver(ordered(t, key, "new", 7, 12)) - ctx, stop := context.WithCancel(context.Background()) - worker := runWorker(ctx, q) - eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) - stop() - <-worker - raw, err := json.Marshal(l.said()[0]) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(raw), `"reads_modules":true`) || !strings.Contains(string(raw), `"reads_generation":true`) { - t.Fatalf("the report does not say it reads modules beside the generation: %s", raw) - } -} diff --git a/internal/link/replay331_test.go b/internal/link/replay331_test.go index 468af9f..087900e 100644 --- a/internal/link/replay331_test.go +++ b/internal/link/replay331_test.go @@ -13,8 +13,8 @@ import ( // The NATS link is attached with no connection under it, so no bus is needed: the question must reach the // link and fail there, not stop at the queue as if no link were open. func TestReplay331(t *testing.T) { - q := &Queue{Membership: Membership{Node: "laptop"}} - q.attach(t.Context(), &natsLink{node: "laptop"}) + q := &Session{Membership: Membership{Node: "laptop"}} + q.Attach(t.Context(), &natsLink{node: "laptop"}) _, err := q.AskTool(t.Context(), "sensors", "sensors_health") if err == nil { t.Fatal("asked on no connection and was answered") diff --git a/internal/link/run.go b/internal/link/run.go index 4628640..3d17258 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -6,18 +6,9 @@ import ( "encoding/json" "errors" "fmt" - "sort" - "strings" "time" ) -// ErrForged is what a node returns for a declaration whose signature is not the mesh's. -// -// Its own error, and it must never be confused with a malformed message. novox/hq ADR 0004 -// requires a host to tell *this is not from the mesh I joined* apart from *this is malformed*: -// the first means somebody is trying, the second means something is broken. -var ErrForged = errors.New("this declaration was not signed by the mesh this node joined") - // Membership is what a node needs to reach its mesh again, held by the caller. type Membership struct { Node string @@ -30,13 +21,6 @@ type Membership struct { Transport string } -// Applier is what the host does with a declaration that has been proved to come from the mesh. -// -// It receives the signature as well as the declaration, so the host can keep both: what it was -// told is kept signed and verified again when it is read back, which means the file on disk is -// trusted for the same reason the message was rather than for being local. -type Applier func(ctx context.Context, declaration, signature []byte) Report - // Announce is how the link says what is happening, so a node running unattended leaves an // account of it. Nil is allowed and means say nothing. type Announce func(string) @@ -64,35 +48,14 @@ type Announce func(string) // Nil is allowed and means nothing ever rouses it, which is every machine that does not suspend. type Roused <-chan struct{} -func Hold(ctx context.Context, m Membership, queue *Queue, say Announce, timeout time.Duration) error { +func Hold(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration) error { return HoldRoused(ctx, m, queue, say, timeout, nil) } -// Unsaid keeps the report of the last apply until the mesh has taken it, so a report lost between -// applying and publishing — the host standing aside for a successor, a crash, a power cut — is said -// again the next time this node is linked (novox/hq issue 264). Without it the declaration has been -// settled or applied, the machine is what the mesh said, and the mesh waits for a report that the -// node believes is gone. -// -// **Exactly the report the apply made, never a new one.** Re-applying the kept declaration would -// describe the machine as it is now; what the mesh waits for is what was done with what it sent. -// Nil is allowed and keeps nothing. -type Unsaid interface { - // Keep holds the report of an apply until it is said. A report naming no declaration — refused - // before it was read — replaces nothing worth saying, and clears what was kept. - Keep(Report) error - // Said forgets the kept report once the broker has taken the report about the same declaration. - Said(declared string) error - // Pending is the report kept and not yet said, if any. - Pending() (Report, bool, error) -} - // HoldRoused is Hold, told when the machine has reason to think its link is stale. // -// **What arrives is enqueued, never applied here** (novox/hq to-be 45 §6). The queue's one worker -// applies, outlives every link, and says its reports on whichever link is open; the caller runs it -// (Queue.Run) for as long as this holds. -func HoldRoused(ctx context.Context, m Membership, queue *Queue, say Announce, +// The session outlives every link and says the node's facts on whichever is open (novox/hq ADR 0539 §9). +func HoldRoused(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration, roused Roused) error { return holdWith(ctx, func(ctx context.Context) error { @@ -185,11 +148,11 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err } } -// Run holds the link open once, applying what arrives and reporting what happened. +// Run holds the link open once, hearing what the controller asks and saying the node's facts. // // Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason; // Hold is what decides whether to open it again. -func Run(ctx context.Context, m Membership, queue *Queue, say Announce, timeout time.Duration) error { +func Run(ctx context.Context, m Membership, queue *Session, say Announce, timeout time.Duration) error { link, err := Open(ctx, m, timeout) if err != nil { return err @@ -200,7 +163,7 @@ func Run(ctx context.Context, m Membership, queue *Queue, say Announce, timeout // serve is Run on a link already open: separated so what the node says, and when, can be tested // without a broker. -func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Announce, +func serve(ctx context.Context, link Link, m Membership, queue *Session, say Announce, timeout time.Duration) error { if say == nil { say = func(string) {} @@ -209,18 +172,13 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou // Said, because it is the event anybody watching actually wants. Without it a node logs every // failure and nothing on success, so a log full of "trying again" and then silence reads as // still broken when it means the opposite. - say("in the mesh, hearing what this node should be") + say("in the mesh, hearing what the controller asks of this node") - // **What the last apply did, if the mesh never heard it**, is said before anything newly - // delivered can be: the queue says it as the link is handed to it (novox/hq issue 264). And the - // link is not let go while the queue's worker is mid-act: an apply that ends the link — the host - // standing aside for a successor it delivered (novox/hq ADR 0141) — is still reported on it, - // before the deferred close. - queue.attach(ctx, link) - defer queue.detach(link) + // The session says this node's facts as the link is handed to it, and is not let go while it is mid-act. + queue.Attach(ctx, link) + defer queue.Detach(link) - declarations := link.Declarations() - // And the mesh asking what this node last applied (to-be 45 §6), on a link that hears the question. + // And the mesh asking this node to say its facts again, on a link that hears the question. var asks <-chan struct{} if a, ok := link.(Asked); ok { asks = a.AskedToReport() @@ -248,7 +206,6 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou } for { - // Asked to stop — by standing aside, say — nothing further is taken in. if ctx.Err() != nil { return nil } @@ -258,90 +215,11 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou case reason := <-link.Lost(): return reason case <-asks: - queue.ReportAsked() - case declaration, ok := <-declarations: - if !ok { - // The link's own reason, when it has managed to say one: "stopped delivering" on - // its own says nothing about why, and why is the whole of what an operator wants. - select { - case reason := <-link.Lost(): - return reason - default: - return errors.New("the mesh stopped sending this node declarations") - } - } - // Whatever else is already waiting goes with it. **Enqueued, not applied**: the queue - // applies the newest of everything delivered and not yet taken, and reports the rest as - // set aside (novox/hq to-be 45 §6). - queue.Deliver(gather(declarations, declaration, drainWindow)...) + queue.FactsAsked(ctx) } } } -// short is a declaration's digest as a person reads it in a log line. -func short(digest string) string { - if len(digest) > 12 { - return digest[:12] - } - return digest -} - -// drainDepth is how many declarations the link holds unread; drainWindow is how long it waits for -// another to follow the one it has. Both small: a push is rare and a backlog is the exception this -// exists for, not the shape of ordinary traffic. -const ( - drainDepth = 16 - drainWindow = 750 * time.Millisecond -) - -// gather takes what is already waiting behind `first`, in the order it arrived. It waits `window` -// for a straggler after each arrival and no longer: a declaration in flight from the mesh arrives -// within that; one that does not is the next push. Which of them is applied is the queue's to decide -// (pick), across everything delivered and not yet taken. -// -// **Its job narrows once declarations are state rather than messages, and does not disappear.** -// On the bus a declaration is last-per-subject (novox/hq design 29 §4), so a node that was away -// receives exactly the current one instead of a queue of superseded ones — the catch-up half is the -// stream's. And a declaration's order decides which is newest, where this window only infers it from -// arrival time, which is the wire-level answer to novox/hq issue 107. -// -// What remains is the live case: three pushes in quick succession to a *connected*, idle node are -// three deliveries, whatever the stream later retains, and gathering them is what makes them one -// apply rather than two. So this is narrowed, not deleted — and saying which half goes is worth more -// than a note that it "can probably be removed", which is how a load-bearing window gets deleted by -// somebody in a hurry. -func gather(arriving <-chan Declaration, first Declaration, window time.Duration) []Declaration { - batch := []Declaration{first} - for { - select { - case next, ok := <-arriving: - if !ok { - return batch - } - batch = append(batch, next) - case <-time.After(window): - return batch - } - } -} - -// handleBody is the whole of deciding whether to trust a message, separated from the broker so it -// can be tested as the security check it is rather than as message plumbing. -func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report { - var signed Signed - if err := json.Unmarshal(body, &signed); err != nil { - return Report{Node: m.Node, Refused: "this message is not a declaration: " + err.Error()} - } - - // Before anything is read out of it, let alone applied. The host applies whatever the link - // delivers, so this check is the difference between the mesh changing this machine and - // anybody changing it. - if !ed25519.Verify(m.Signer, signed.Declaration, signed.Signature) { - return Report{Node: m.Node, Refused: ErrForged.Error()} - } - return apply(ctx, signed.Declaration, signed.Signature) -} - // publishReport tells the mesh what this node did, and says whether the broker took it. // Publish sends one report on this node's own connection and returns: the one-shot path for a // report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same @@ -379,36 +257,3 @@ func publishReport(ctx context.Context, bus Bus, m Membership, report Report, } return true } - -// heldNote is what this apply did NOT do, for the line that says what it did. -// -// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125). -// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but -// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources, -// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took -// opening state.json by hand; not reading it took every public name on the machine down, because the -// operator had four green surfaces and a discrepancy nobody could interpret. -// -// So the line that reports the apply carries it. Grouped by module and ordered by name, because the -// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing -// they can act on — `take` is the verb, and it takes a module. -func heldNote(held []Held) string { - if len(held) == 0 { - return "" - } - byModule := map[string]int{} - for _, h := range held { - byModule[h.Module]++ - } - names := make([]string, 0, len(byModule)) - for name := range byModule { - names = append(names, name) - } - sort.Strings(names) - parts := make([]string, 0, len(names)) - for _, name := range names { - parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name])) - } - return fmt.Sprintf(", %d held until their module is taken (%s)", - len(held), strings.Join(parts, ", ")) -} diff --git a/internal/link/session.go b/internal/link/session.go new file mode 100644 index 0000000..b90f733 --- /dev/null +++ b/internal/link/session.go @@ -0,0 +1,310 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program +// that wrote them, beside its epoch and sequence. +const Writer = "node-engine" + +// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks +// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts. +// +// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an +// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles +// a whole declaration. +type Session struct { + // Membership is the node and the mesh's key a controller's ask is verified with. + Membership Membership + // OnLink is run, off the link's own loop, each time a link opens and when the controller asks the node to say its + // facts again. It says them with SayFacts. Nil says nothing. + OnLink func(ctx context.Context) + // Heard is told every facts event the mesh has taken. Nil is allowed. + Heard func(Report) + // HaveWritten is told each time a have key is written, or found already saying what this node-engine would write: + // the first proves a node-engine on trial (novox/hq ADR 0539 §9). Nil is allowed. + HaveWritten func() + // HandOver judges and records a directory used as found handed to the mesh by the operator, asked by + // the controller over the link (novox/hq issue 356). Nil answers every such ask with a refusal. + HandOver HandOverFunc + // SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the + // controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal. + SetuidSearch SetuidSearchFunc + // JudgeDirectories judges this node's declared directories read-only, asked by the controller over the + // link (novox/hq issue 358). Nil answers every such ask with a refusal. + JudgeDirectories JudgeDirectoriesFunc + // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved + // warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal. + AccessAct AccessActFunc + // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of + // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. + MoveSecret MoveSecretFunc + Say Announce + Timeout time.Duration + + once sync.Once + + mu sync.Mutex + bus Bus // the link open now; nil while there is none + // gone is closed when the link open now closes, and changed whenever a link opens or closes (WhenLinked). + gone chan struct{} + changed chan struct{} + + // saying is held while a facts event is made and said, so they leave in the order they are made. + saying sync.Mutex + // epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one + // without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2). + epoch int64 + sequence int64 + + // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). + haves haveState +} + +func (q *Session) init() { + q.once.Do(func() { + if q.Say == nil { + q.Say = func(string) {} + } + q.epoch = time.Now().UnixMilli() + q.changed = make(chan struct{}) + }) +} + +// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them. +// A facts event made with no link open is not kept: the next link says the facts as they are then. +func (q *Session) SayFacts(ctx context.Context, r Report) bool { + q.init() + q.saying.Lock() + defer q.saying.Unlock() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return false + } + q.sequence++ + r.Node = q.Membership.Node + r.Writer = Writer + r.Order = Order{Epoch: q.epoch, Sequence: q.sequence} + if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) { + return false + } + // After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132). + if q.noteApply(r) { + q.writeHaves(ctx, bus) + } + if q.Heard != nil && r.Refused == "" { + q.Heard(r) + } + return true +} + +// FactsAsked is the mesh asking this node to say its facts again: run on the link open now. +func (q *Session) FactsAsked(ctx context.Context) { + q.init() + if q.OnLink != nil { + go q.OnLink(ctx) + } +} + +// Attach is a link opening: it is the link facts are said on from now. +func (q *Session) Attach(ctx context.Context, bus Bus) { + q.init() + q.mu.Lock() + q.bus = bus + q.gone = make(chan struct{}) + close(q.changed) + q.changed = make(chan struct{}) + q.mu.Unlock() + q.FactsAsked(ctx) +} + +// WithLink is ctx carrying the link now open, so an apply can fetch a composition's files given by digest (novox/hq ADR +// 0486, issue 1695). A session with no link open carries none. +func (q *Session) WithLink(ctx context.Context) context.Context { + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return ctx + } + return WithFiles(ctx, bus) +} + +// Detach is the link closing: nothing more is said on it. +func (q *Session) Detach(bus Bus) { + q.init() + q.saying.Lock() + defer q.saying.Unlock() + q.mu.Lock() + if q.bus == bus { + q.bus = nil + close(q.gone) + close(q.changed) + q.changed = make(chan struct{}) + } + q.mu.Unlock() +} + +// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the +// machine's networking its engine judges). +func (q *Session) Linked() bool { + q.init() + q.mu.Lock() + defer q.mu.Unlock() + return q.bus != nil +} + +// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether +// the bus took it. A statement is a word about the machine as it is, and it must not wait behind an apply: a +// container crash-looping while a long apply runs is said while it runs. False with no link open, or one that cannot +// say it; the caller says it again on its next look, and the next facts event carries it whatever happens. +func (q *Session) SayHealth(ctx context.Context, h Health) bool { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + sayer, ok := bus.(HealthBus) + if bus == nil || !ok { + return false + } + body, err := json.Marshal(HealthSaid{Node: q.Membership.Node, Health: h}) + if err != nil { + return false + } + saying, cancel := context.WithTimeout(ctx, q.timeout()) + defer cancel() + if err := sayer.Health(saying, q.Membership.Node, body); err != nil { + q.Say("could not say this machine's health: " + err.Error()) + return false + } + // A look that changed a module's health changes its have (issue 1132). + if q.noteHealth(h) { + q.writeHaves(ctx, bus) + } + return true +} + +func (q *Session) timeout() time.Duration { + if q.Timeout <= 0 { + return 10 * time.Second + } + return q.Timeout +} + +// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR +// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open +// or nothing answered — said as the check's failure, never as healthy. +func (q *Session) AskTool(ctx context.Context, module, tool string) (health.Answer, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return health.Answer{}, errors.New("no link to the bus is open") + } + asker, ok := bus.(ToolBus) + if !ok { + return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool") + } + reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}")) + if err != nil { + return health.Answer{}, err + } + return ReadToolAnswer(reply) +} + +// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over +// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for. +func (q *Session) AskController(ctx context.Context, body []byte) ([]byte, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + asker, ok := bus.(ToolBus) + if !ok { + return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked) + } + return asker.Ask(ctx, CLISubject(q.Membership.Node), body) +} + +// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any +// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272). +var ErrNotAsked = errors.New("not asked") + +// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call — +// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the +// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the +// SDK cannot check is unhealthy too, so a malformed one never excuses anything. +func ReadToolAnswer(reply []byte) (health.Answer, error) { + var env struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + if err := json.Unmarshal(reply, &env); err != nil { + return health.Answer{}, fmt.Errorf("its answer is not one: %w", err) + } + if env.Error != "" { + return health.Unhealthy(env.Error), nil + } + a, err := health.Read(env.Result) + if err != nil { + return health.Unhealthy("it answered something other than healthy or not"), nil + } + return a, nil +} + +// SeatToolSubject is where a node seat's verb is asked of one machine's holder (design 33 §4): the seat's verb subject +// with the machine as its last token. +func SeatToolSubject(seat, verb, node string) string { + return "mesh.seat." + seat + ".tool." + verb + "." + node +} + +// AskSeat asks the holder of a seat on this node a verb, over the link open now, and answers its result, or its refusal +// as an error. The one call the node agent makes of another part: the backup before a data module moves (ADR 0429 §14). +func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, within time.Duration) (json.RawMessage, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + asker, ok := bus.(ToolBus) + if !ok { + return nil, fmt.Errorf("%w: the link to the bus open now cannot ask a seat", ErrNotAsked) + } + body, err := json.Marshal(args) + if err != nil { + return nil, err + } + asking, cancel := context.WithTimeout(ctx, within) + defer cancel() + reply, err := asker.Ask(asking, SeatToolSubject(seat, verb, q.Membership.Node), body) + if err != nil { + return nil, fmt.Errorf("%s.%s could not be asked: %w", seat, verb, err) + } + var env struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + if err := json.Unmarshal(reply, &env); err != nil { + return nil, fmt.Errorf("%s.%s answered with something that is not a tool's answer", seat, verb) + } + if env.Error != "" { + return nil, errors.New(env.Error) + } + return env.Result, nil +} diff --git a/internal/link/session_test.go b/internal/link/session_test.go new file mode 100644 index 0000000..909ed82 --- /dev/null +++ b/internal/link/session_test.go @@ -0,0 +1,164 @@ +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled, +// as the bus's is ("reporting: context canceled"), or while refusing. +type quietLink struct { + mu sync.Mutex + reports []Report + refusing bool + lost chan error +} + +func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} } + +func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error { + if err := ctx.Err(); err != nil { + return err + } + l.mu.Lock() + defer l.mu.Unlock() + if l.refusing { + return errors.New("refused") + } + var r Report + if err := json.Unmarshal(body, &r); err != nil { + return err + } + l.reports = append(l.reports, r) + return nil +} +func (l *quietLink) Lost() <-chan error { return l.lost } +func (l *quietLink) Close() {} + +func (l *quietLink) said() []Report { + l.mu.Lock() + defer l.mu.Unlock() + return append([]Report(nil), l.reports...) +} + +func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { + t.Helper() + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return Membership{Node: "n1", Signer: public}, private +} + +// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539 +// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts. +func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) { + m, _ := aMember(t) + l := newQuietLink() + s := &Session{Membership: m, Timeout: time.Second} + s.Attach(context.Background(), l) + + if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) { + t.Fatal("the facts were not taken") + } + got := l.said() + if len(got) != 2 { + t.Fatalf("%d facts events were said", len(got)) + } + for _, r := range got { + if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 { + t.Fatalf("the facts do not say who wrote them: %+v", r) + } + } + if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 { + t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order) + } + + // A later start orders after this one with nothing kept on disk. + time.Sleep(2 * time.Millisecond) + later := &Session{Membership: m, Timeout: time.Second} + later.Attach(context.Background(), l) + later.SayFacts(context.Background(), Report{}) + if last := l.said()[2]; last.Epoch <= got[1].Epoch { + t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch) + } +} + +// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then. +func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { + m, _ := aMember(t) + l := newQuietLink() + s := &Session{Membership: m, Timeout: time.Second} + if s.SayFacts(context.Background(), Report{}) { + t.Fatal("facts were said with no link") + } + s.Attach(context.Background(), l) + s.Detach(l) + if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 { + t.Fatal("facts were said on a link that was let go") + } +} + +// A link opening says the node's facts, off the link's own loop. +func TestALinkOpeningSaysTheFacts(t *testing.T) { + m, _ := aMember(t) + l := newQuietLink() + opened := make(chan struct{}, 1) + s := &Session{Membership: m, Timeout: time.Second, OnLink: func(context.Context) { opened <- struct{}{} }} + s.Attach(context.Background(), l) + select { + case <-opened: + case <-time.After(2 * time.Second): + t.Fatal("a link opening did not ask for the facts") + } +} + +// aBus is a connection to a real server, because some claims here are about one: set MESH_TEST_NATS to a server with +// JetStream, or the test is skipped. +func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, + Retention: nats.WorkQueuePolicy} + if _, err := js.StreamInfo(want.Name); err != nil { + if _, err := js.AddStream(want); err != nil { + t.Fatal(err) + } + } + if err := js.PurgeStream(want.Name); err != nil { + t.Fatal(err) + } + return conn, js +} + +// eventuallyHeld waits for a condition the bus reaches on its own time. +func eventuallyHeld(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatal(what) + } + time.Sleep(5 * time.Millisecond) + } +} diff --git a/internal/link/unsaid_test.go b/internal/link/unsaid_test.go deleted file mode 100644 index 13257fd..0000000 --- a/internal/link/unsaid_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package link - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "errors" - "reflect" - "sync" - "testing" - "time" -) - -// A report of an apply reaches the mesh even when the apply ended the link, and one lost anyway is -// said again the next time the node is linked (novox/hq issue 264). - -// quietLink is a link with no broker behind it. A report is refused once the context it is -// published on is cancelled, as the bus's is ("reporting: context canceled"), or while refusing. -type quietLink struct { - mu sync.Mutex - reports []Report - refusing bool - delivered chan Declaration - lost chan error -} - -func newQuietLink(arriving ...Declaration) *quietLink { - l := &quietLink{delivered: make(chan Declaration, len(arriving)), lost: make(chan error, 1)} - for _, d := range arriving { - l.delivered <- d - } - return l -} - -func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error { - if err := ctx.Err(); err != nil { - return err - } - l.mu.Lock() - defer l.mu.Unlock() - if l.refusing { - return errors.New("refused") - } - var r Report - if err := json.Unmarshal(body, &r); err != nil { - return err - } - l.reports = append(l.reports, r) - return nil -} -func (l *quietLink) Declarations() <-chan Declaration { return l.delivered } -func (l *quietLink) Lost() <-chan error { return l.lost } -func (l *quietLink) Close() {} - -func (l *quietLink) said() []Report { - l.mu.Lock() - defer l.mu.Unlock() - return append([]Report(nil), l.reports...) -} - -// keptInMemory is Unsaid without a disk. -type keptInMemory struct { - mu sync.Mutex - report *Report -} - -func (k *keptInMemory) Keep(r Report) error { - k.mu.Lock() - defer k.mu.Unlock() - if r.Declared == "" { - k.report = nil - return nil - } - k.report = &r - return nil -} -func (k *keptInMemory) Said(declared string) error { - k.mu.Lock() - defer k.mu.Unlock() - if k.report != nil && k.report.Declared == declared { - k.report = nil - } - return nil -} -func (k *keptInMemory) Pending() (Report, bool, error) { - k.mu.Lock() - defer k.mu.Unlock() - if k.report == nil { - return Report{}, false, nil - } - return *k.report, true, nil -} - -func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { - t.Helper() - public, private, err := ed25519.GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - return Membership{Node: "n1", Signer: public}, private -} - -// Measured on the anchor: a declaration carried a new controller and a new host, the host applied -// it, stood aside for its successor — which cancels the link — and the report of that apply failed -// with "context canceled". The plan waited on it until somebody pushed by hand. -// -// R3 of novox/hq to-be 45 §9, as a unit: the node-engine self-updates during its report, and the -// report still arrives — on the link the apply came over, before it is let go. -func TestAnApplyThatEndsTheLinkIsStillReported(t *testing.T) { - m, key := aMember(t) - declaration := &said{body: signedBy(t, key, []byte(`{"declaration":1}`))} - behind := &said{body: signedBy(t, key, []byte(`{"declaration":2}`))} - l := newQuietLink(declaration) - kept := &keptInMemory{} - - ctx, standAside := context.WithCancel(context.Background()) - defer standAside() - applied := 0 - q := aQueue(m, func(context.Context, []byte, []byte) Report { - applied++ - standAside() // the host delivered its successor, and stands aside for it - return Report{Declared: "d1", Applied: []string{"a"}} - }, kept) - worker := runWorker(ctx, q) - - done := make(chan error, 1) - go func() { done <- serve(ctx, l, m, q, nil, time.Second) }() - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(5 * time.Second): - t.Fatal("the link did not end after the apply stood aside") - } - // Something delivered after the host stood aside is not applied by it: the successor will be - // sent it again. - q.Deliver(behind) - <-worker - - reports := l.said() - if len(reports) != 1 || reports[0].Declared != "d1" || !reflect.DeepEqual(reports[0].Applied, []string{"a"}) { - t.Fatalf("the apply's report did not reach the mesh: %+v", reports) - } - if !declaration.wasHandled() { - t.Fatal("the declaration was not settled after its report") - } - if applied != 1 || behind.wasHandled() { - t.Fatalf("%d applies, and the declaration behind settled %v: after standing aside nothing "+ - "further is applied", applied, behind.wasHandled()) - } - if _, ok, _ := kept.Pending(); ok { - t.Fatal("a report the mesh took is still kept to be said again") - } -} - -// A report that did not reach the mesh — the host died between applying and publishing, or the -// broker refused it — is said on the next link, before anything else, and exactly as it was made. -func TestAReportLostAfterTheApplyIsSaidOnTheNextLink(t *testing.T) { - m, key := aMember(t) - made := Report{Declared: "d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"}} - kept := &keptInMemory{} - - // The apply happens and its report is lost. - first := newQuietLink(&said{body: signedBy(t, key, []byte(`{"declaration":1}`))}) - first.refusing = true - ctx, stop := context.WithCancel(context.Background()) - q := aQueue(m, func(context.Context, []byte, []byte) Report { stop(); return made }, kept) - worker := runWorker(ctx, q) - if err := serve(ctx, first, m, q, nil, time.Second); err != nil { - t.Fatal(err) - } - <-worker - if len(first.said()) != 0 { - t.Fatal("the broker refused the report and it counted as said") - } - lost, ok, _ := kept.Pending() - if !ok || lost.ReportSequence != 1 { - t.Fatalf("the lost report was not kept as it was made: %+v", lost) - } - - // The next host links. It is stopped at once, so all it does is what it does on linking. - next := newQuietLink() - gone, cancel := context.WithCancel(context.Background()) - cancel() - never := func(context.Context, []byte, []byte) Report { - t.Fatal("nothing was delivered, and something was applied") - return Report{} - } - if err := serve(gone, next, m, aQueue(m, never, kept), nil, time.Second); err != nil { - t.Fatal(err) - } - reports := next.said() - if len(reports) != 1 || !reflect.DeepEqual(reports[0], lost) { - t.Fatalf("the lost report was not said again as it was made: %+v", reports) - } - if _, ok, _ := kept.Pending(); ok { - t.Fatal("a report said again is still kept") - } -} - -// A node that never applied anything, or whose last report was taken, says nothing on linking. -func TestNothingIsSaidAgainWhenNothingWasLost(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - gone, cancel := context.WithCancel(context.Background()) - cancel() - if err := serve(gone, l, m, aQueue(m, nil, &keptInMemory{}), nil, time.Second); err != nil { - t.Fatal(err) - } - if err := serve(gone, l, m, aQueue(m, nil, nil), nil, time.Second); err != nil { - t.Fatal(err) - } - if reports := l.said(); len(reports) != 0 { - t.Fatalf("a node with nothing lost reported %+v", reports) - } -} diff --git a/internal/link/watch.go b/internal/link/watch.go new file mode 100644 index 0000000..0a78e6e --- /dev/null +++ b/internal/link/watch.go @@ -0,0 +1,176 @@ +package link + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go" +) + +// KVUpdate is one change of a key in a bucket the node-engine watches (novox/hq ADR 0539 §9). Caught marks the end of +// the keys that were there when the watch began. +type KVUpdate struct { + Key string + Value []byte + Revision uint64 + Deleted bool + Caught bool +} + +// KVBus is a link that can watch a bucket's keys: the node agent follows its wants and its compositions with it, and +// never polls. +type KVBus interface { + // KVWatch yields the keys of the bucket matching the pattern now, a Caught update, then every change, until ctx ends + // or the link does (the channel is then closed). A bucket the node's account may not read is an error. + KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error) +} + +// KVWatch is the watch on the link open now. +func (b OverNATS) KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error) { + js := b.JS + if b.Conn != nil { + bound, err := b.Conn.JetStream() + if err != nil { + return nil, err + } + js = bound + } + kv, err := js.KeyValue(bucket) + if err != nil { + return nil, fmt.Errorf("the bucket %s: %w", bucket, err) + } + w, err := kv.Watch(pattern, nats.Context(ctx)) + if err != nil { + return nil, fmt.Errorf("watching %s in %s: %w", pattern, bucket, err) + } + out := make(chan KVUpdate, 64) + go func() { + defer close(out) + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + var u KVUpdate + if e == nil { + u.Caught = true + } else { + u = KVUpdate{Key: e.Key(), Value: e.Value(), Revision: e.Revision(), + Deleted: e.Operation() == nats.KeyValueDelete || e.Operation() == nats.KeyValuePurge} + } + select { + case out <- u: + case <-ctx.Done(): + return + } + } + } + }() + return out, nil +} + +func (l *natsLink) KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error) { + return OverNATS{Conn: l.conn, JS: l.js}.KVWatch(ctx, bucket, pattern) +} + +var ( + _ KVBus = OverNATS{} + _ KVBus = (*natsLink)(nil) +) + +// ErrNoLink is what a watch asks for with no link open. +var ErrNoLink = errors.New("no link to the bus is open") + +// KVWatch watches a bucket on the link open now. It ends when ctx does or the link closes. +func (q *Session) KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + w, ok := bus.(KVBus) + if bus == nil || !ok { + return nil, ErrNoLink + } + return w.KVWatch(ctx, bucket, pattern) +} + +// WhenLinked waits for a link to be open and returns a context that ends when that link does (or ctx does): what the +// node agent runs under, so its watches end with the link and begin again with the next. +func (q *Session) WhenLinked(ctx context.Context) (context.Context, context.CancelFunc, error) { + q.init() + for { + q.mu.Lock() + if q.bus != nil { + gone := q.gone + q.mu.Unlock() + linked, cancel := context.WithCancel(ctx) + go func() { + select { + case <-gone: + cancel() + case <-linked.Done(): + } + }() + return linked, cancel, nil + } + changed := q.changed + q.mu.Unlock() + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-changed: + } + } +} + +// EventBus is a link that can hear an event on a subject of core NATS: the node agent waits for a backup's end with it, +// and never polls. +type EventBus interface { + // Hear yields the payloads published on the subject until stop is called or the link ends. + Hear(subject string) (events <-chan []byte, stop func(), err error) +} + +func (b OverNATS) Hear(subject string) (<-chan []byte, func(), error) { + out := make(chan []byte, 16) + sub, err := b.Conn.Subscribe(subject, func(m *nats.Msg) { + select { + case out <- m.Data: + default: + } + }) + if err != nil { + return nil, nil, err + } + if err := b.Conn.Flush(); err != nil { + _ = sub.Unsubscribe() + return nil, nil, err + } + return out, func() { _ = sub.Unsubscribe() }, nil +} + +func (l *natsLink) Hear(subject string) (<-chan []byte, func(), error) { + return OverNATS{Conn: l.conn, JS: l.js}.Hear(subject) +} + +var ( + _ EventBus = OverNATS{} + _ EventBus = (*natsLink)(nil) +) + +// Hear listens for an event on the link open now, from before the caller asks for what it waits on. +func (q *Session) Hear(subject string) (<-chan []byte, func(), error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + h, ok := bus.(EventBus) + if bus == nil || !ok { + return nil, nil, ErrNoLink + } + return h.Hear(subject) +} diff --git a/internal/link/witnessing.go b/internal/link/witnessing.go index 340eaec..4b1f8ad 100644 --- a/internal/link/witnessing.go +++ b/internal/link/witnessing.go @@ -48,7 +48,7 @@ func DirectGetSubject(bucket, key string) string { func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id } // Asker is the link open now, when there is one and it can ask. -func (q *Queue) Asker() Asker { +func (q *Session) Asker() Asker { q.init() q.mu.Lock() defer q.mu.Unlock() diff --git a/internal/profile/detectors.go b/internal/profile/detectors.go index aedfde2..0f461ba 100644 --- a/internal/profile/detectors.go +++ b/internal/profile/detectors.go @@ -40,6 +40,11 @@ const ( // controller sends the field only to a node that reports it, since an older node-engine refuses the key. CapContainerGroups = "container-groups" + // CapReadsWants says this node-engine follows its wants (novox/hq ADR 0539 §9): it watches `want..>` and its + // compositions and applies one module at a time, and hears no declaration. A fact about the program: the controller + // stops sending a node declarations only once it reports it. + CapReadsWants = "reads-wants" + // The network manager this machine runs, one capability per dialect (novox/hq ADR 0161): the // uplink seat's holder declares its own, so the holder for a manager the machine does not run // is refused the way any missing capability is, naming it. Active, not installed — a machine @@ -190,6 +195,20 @@ func (containerGroups) Detect(context.Context) Verdict { } } +// readsWants reports that this node-engine follows its wants. True of the program that is running, so it is always +// present: its absence on a node is what tells the controller the node-engine still waits for declarations. +type readsWants struct{} + +func (readsWants) Name() string { return CapReadsWants } + +func (readsWants) Detect(context.Context) Verdict { + return Verdict{ + Name: CapReadsWants, Present: true, + Detail: "follows its wants and applies one module's composition at a time; hears no declaration", + How: "a fact about this program, not probed: the node-engine that answers follows its wants", + } +} + // graphicalSession reports whether anything could display a window here. // // Environment rather than a probe, because a display server is reachable through a socket a @@ -230,6 +249,7 @@ func Default(runner Runner) []Detector { privileged{}, fileReferences{}, containerGroups{}, + readsWants{}, graphicalSession{}, seat{}, battery{}, diff --git a/internal/profile/reads_wants_test.go b/internal/profile/reads_wants_test.go new file mode 100644 index 0000000..fb77378 --- /dev/null +++ b/internal/profile/reads_wants_test.go @@ -0,0 +1,22 @@ +package profile + +import ( + "context" + "testing" + "time" +) + +// R6-A: the node's profile carries the capability that says the node-engine follows its wants, which the controller +// checks before it stops sending the node declarations (novox/hq ADR 0539 §9). +func TestTheProfileSaysTheNodeEngineReadsItsWants(t *testing.T) { + got := Detect(context.Background(), Default(func(context.Context, string, ...string) (string, error) { + return "", nil + }), time.Second) + v := verdictOf(t, got, CapReadsWants) + if !v.Present || v.Detail == "" || v.How == "" { + t.Errorf("reads-wants = %+v, want present with a detail and a method", v) + } + if CapReadsWants != "reads-wants" { + t.Errorf("the capability is named %q", CapReadsWants) + } +} diff --git a/internal/store/numbers.go b/internal/store/numbers.go deleted file mode 100644 index 61fe263..0000000 --- a/internal/store/numbers.go +++ /dev/null @@ -1,94 +0,0 @@ -package store - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" -) - -// What the node-engine counts about what it says, kept so the counting outlives the process. -// -// **The report sequence must increase across restarts and self-updates** (novox/hq to-be 45 §6). The -// mesh keeps, per machine, the highest report it accepted and refuses an older one; a host that began -// again from one after every restart — and a self-update is a restart — would have every report it -// made refused until it had counted past where its predecessor stopped. So the number is kept beside -// the state, where the successor reads it. -// -// And the count of declarations refused as older than what this node applied (rule 2), which the -// mesh's stale-writer watchdog reads from every report. - -// NumbersName is where they live, beside the state. -const NumbersName = "numbers.json" - -// NumbersPath is where the numbers live, given where the state lives. -func NumbersPath(statePath string) string { - return filepath.Join(filepath.Dir(statePath), NumbersName) -} - -// Numbers is what is kept. -type Numbers struct { - // ReportSequence is the number of the last report this node made. - ReportSequence int64 `json:"report_sequence"` - // RefusedOlder is how many declarations it has refused as older, ever. - RefusedOlder int64 `json:"refused_older"` -} - -// ReadNumbers is what was kept, or zero when nothing was — a node that has never reported. -// -// **Unreadable is an error, never zero** (novox/hq ADR 0227, rule 4): read as zero, every report the -// node makes next would be older than the ones it already made, and refused. -func ReadNumbers(path string) (Numbers, error) { - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return Numbers{}, nil - } - if err != nil { - return Numbers{}, err - } - var n Numbers - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.DisallowUnknownFields() - if err := dec.Decode(&n); err != nil { - return Numbers{}, fmt.Errorf("the numbers kept at %s are unreadable: %w", path, err) - } - if n.ReportSequence < 0 || n.RefusedOlder < 0 { - return Numbers{}, fmt.Errorf("the numbers kept at %s are below zero, which nothing counting writes", path) - } - return n, nil -} - -// SaveNumbers keeps them, replacing what was kept, and is on disk when it returns: a number used and -// not kept is one the next host would use again. -func SaveNumbers(path string, n Numbers) error { - raw, err := json.Marshal(n) - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err - } - tmp, err := os.CreateTemp(filepath.Dir(path), ".numbers-*") - if err != nil { - return err - } - defer os.Remove(tmp.Name()) - if err := tmp.Chmod(0o600); err != nil { - tmp.Close() - return err - } - if _, err := tmp.Write(raw); err != nil { - tmp.Close() - return err - } - if err := tmp.Sync(); err != nil { - tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(tmp.Name(), path) -} diff --git a/internal/store/store.go b/internal/store/store.go index 3726606..5755b81 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -34,6 +34,10 @@ const DefaultPath = "/var/lib/mesh-host/state.json" type Applied struct { ID string `json:"id"` Type string `json:"type"` + // Module is the module whose want this resource was applied for (novox/hq ADR 0429 §4, ADR 0539 §9): the owner + // record that limits a removal to the module whose want changed. Absent on a record from before; then the + // module is read from the id (OwnerOf). + Module string `json:"module,omitempty"` // Origin is who asked for this: the bundle this host carries, or the mesh. // // Recorded because the two must not remove each other. A node raises its own foundation from @@ -680,6 +684,33 @@ func (s State) Orphans(declared map[string]bool, origin string) []Applied { return out } +// NodeModule is the owner of what belongs to no module: the node's own resources (adoption, firewall openings, +// accounts, the resolver, identities and groups; novox/hq ADR 0539 §2). +const NodeModule = "node" + +// OwnerOf is the module a record belongs to: the one recorded with it, else the one its id names, else the node. +// guess reads a module from an id (declaration.ModuleGuess), kept as a parameter so the store imports nothing. +func OwnerOf(a Applied, guess func(id string) (string, bool)) string { + if a.Module != "" { + return a.Module + } + if guess != nil { + if m, ok := guess(a.ID); ok { + return m + } + } + return NodeModule +} + +// SetModule records which module a resource was applied for. +func (s *State) SetModule(id, module string) { + for i := range s.Resources { + if s.Resources[i].ID == id { + s.Resources[i].Module = module + } + } +} + // Origins a resource can have. const ( // Carried is the bundle this host was built with. diff --git a/internal/store/unsaid.go b/internal/store/unsaid.go deleted file mode 100644 index 4412b19..0000000 --- a/internal/store/unsaid.go +++ /dev/null @@ -1,76 +0,0 @@ -package store - -import ( - "errors" - "os" - "path/filepath" -) - -// The report of the last apply of a declaration from the mesh, kept until the mesh has taken it. -// -// **An apply the mesh never heard about is one it waits on for ever.** The controller's release -// plan waits for a machine to report the exact declaration it was sent; a report lost between the -// apply and the publish — a host standing aside for its successor, a crash, a power cut — leaves the -// machine applied and the plan waiting until somebody pushes by hand (novox/hq issue 264). The -// declaration itself is kept (declared.go), and re-applying it would say something new about the -// machine; what was lost is what *that* apply did, so that is what is kept, exactly as it was made, -// and said again once the host is linked. -// -// Opaque bytes here: the report is the link's word, and the store keeps it without reading it. - -// UnsaidName is where it lives, beside the state. -const UnsaidName = "unsaid.json" - -// UnsaidPath is where the unsaid report lives, given where the state lives. -func UnsaidPath(statePath string) string { - return filepath.Join(filepath.Dir(statePath), UnsaidName) -} - -// SaveUnsaid keeps a report the mesh has not taken yet, replacing whatever was kept before: only -// the last apply's report is worth saying again, because the mesh compares against what it sent last. -func SaveUnsaid(path string, report []byte) error { - if len(report) == 0 { - return errors.New("refusing to keep an empty report") - } - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err - } - tmp, err := os.CreateTemp(filepath.Dir(path), ".unsaid-*") - if err != nil { - return err - } - defer os.Remove(tmp.Name()) - if err := tmp.Chmod(0o600); err != nil { - tmp.Close() - return err - } - if _, err := tmp.Write(report); err != nil { - tmp.Close() - return err - } - if err := tmp.Sync(); err != nil { - tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(tmp.Name(), path) -} - -// ReadUnsaid is the report kept unsaid, or nil when there is none — the ordinary case. -func ReadUnsaid(path string) ([]byte, error) { - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } - return raw, err -} - -// ClearUnsaid forgets the kept report, once the mesh has taken it. Absent is already clear. -func ClearUnsaid(path string) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return nil -} diff --git a/internal/upgrade/witnessed.go b/internal/upgrade/witnessed.go index 4a2cf03..5b386de 100644 --- a/internal/upgrade/witnessed.go +++ b/internal/upgrade/witnessed.go @@ -87,7 +87,7 @@ func RolledBackVersions(verdicts []Verdict) []string { return out } -// Proved is this host's version seen to report its declaration under its own build: the evidence a +// Proved is this host's version seen to write a have key under its own build (novox/hq ADR 0539 §9; before it was a report of a declaration): the evidence a // trial waits for, and the only evidence known-good has ever claimed (novox/hq ADR 0005). // // - known-good becomes this version, which ends the launcher's trial; diff --git a/module.json b/module.json index ffa13e3..fcb6b77 100644 --- a/module.json +++ b/module.json @@ -14,6 +14,16 @@ } ] }, + "consumes": [ + "restic.backed-up" + ], + "config": { + "operator-signers": { + "type": "string", + "description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.", + "default": "" + } + }, "resources": [ { "id": "launcher", @@ -27,6 +37,13 @@ "type": "archive", "artifact": "host-arch", "path": "/usr/lib/nox-mesh-host/versions/${version}" + }, + { + "id": "operator-keys", + "type": "file", + "path": "/var/lib/mesh-host/operator-keys", + "mode": "0644", + "content": "${config:operator-signers}\n" } ] } diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go new file mode 100644 index 0000000..5935047 --- /dev/null +++ b/packaging/operator_keys_test.go @@ -0,0 +1,71 @@ +package packaging_test + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The operator's public keys reach every node as a file beside the state file, fed by a setting of the node-engine +// (novox/hq ADR 0539 clauses 3 and 11): the node agent reads it, and a composition made by hand is applied only when one +// of these keys signed it. The setting may be empty, and then no operator key is trusted. +func TestTheManifestDeclaresTheOperatorKeysFileFedByASetting(t *testing.T) { + raw, err := os.ReadFile("../module.json") + if err != nil { + t.Fatal(err) + } + if _, err := catalogue.ParseManifest(raw); err != nil { + t.Fatalf("the manifest no longer parses: %v", err) + } + var m struct { + Config map[string]struct { + Type string `json:"type"` + Default any `json:"default"` + } `json:"config"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Path string `json:"path"` + Mode string `json:"mode"` + Content string `json:"content"` + } `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + setting, ok := m.Config["operator-signers"] + if !ok || setting.Type != "string" || setting.Default != "" { + t.Fatalf("the setting operator-signers is %+v: a string that may be empty", setting) + } + for _, r := range m.Resources { + if r.ID != "operator-keys" { + continue + } + if r.Type != "file" || r.Path != "/var/lib/mesh-host/operator-keys" || r.Mode != "0644" || + !strings.Contains(r.Content, "${config:operator-signers}") { + t.Fatalf("the file is %+v: it is declared beside the state file, 0644, its content the setting", r) + } + return + } + t.Fatal("module.json declares no operator-keys file") +} + +// The node agent hears the end of a backup on the event of node-backup's holder, so the manifest consumes it. +func TestTheManifestConsumesTheBackupEvent(t *testing.T) { + raw, err := os.ReadFile("../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Consumes []string `json:"consumes"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" { + t.Fatalf("consumes %v", m.Consumes) + } +}