From 5dc3a8e17f97094d5d31dee07af886883998a2ab Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 10:52:42 +0100 Subject: [PATCH] Keep what an archive unpacked in the engine's own state directory, never beside the archive, and stop no slower than the host waits (issue 1585, review round 1) The journal of an unpack sat at .unpacked.json, in a directory the module's account may write: a forged one would make the root engine remove or adopt any path it named. It now lives under /unpacked/.json, mode 0600, names its target, and is refused whole when an entry is not filepath.IsLocal or a parent is not above the target. It goes when the archive is taken off, whether its state record still said what it unpacked or not. The launcher's stop grace for a host on trial rises from 20 s to 75 s and the unit's TimeoutStopSec is 120 s (openrc the same), past the host's own 60 s wait for the apply in flight, so neither cuts a save off again. Tests folded as rows into existing ones (the repository is over its unit budget): the journal's rows, a forged entry and parent refused, the journal gone with a record that lost its Unpacked; the test that passed on main dropped; the stop test waits on its own line, not a sleep, and no longer consumes the save it waits for. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- cmd/mesh-host/main_test.go | 22 ++++- internal/apply/apply.go | 8 +- internal/apply/archive.go | 9 +- internal/apply/archive_journal.go | 121 +++++++++++++++++++------ internal/apply/archive_journal_test.go | 115 ----------------------- internal/apply/archive_removal_test.go | 93 +++++++++++++++++++ internal/apply/archive_replace_test.go | 17 +--- internal/apply/archive_unpacked.go | 11 ++- internal/apply/home_links_test.go | 2 +- module.json | 2 +- packaging/nox-mesh-host-launch | 4 +- packaging/nox-mesh-host.openrc | 2 + packaging/nox-mesh-host.service | 3 + 13 files changed, 234 insertions(+), 175 deletions(-) delete mode 100644 internal/apply/archive_journal_test.go diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 72a43f6..cd4df78 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -140,15 +140,25 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { applying.Lock() var said []string var mu sync.Mutex - say := func(line string) { mu.Lock(); said = append(said, line); mu.Unlock() } + waiting := make(chan struct{}) + say := func(line string) { + mu.Lock() + said = append(said, line) + mu.Unlock() + if strings.Contains(line, "waiting up to") { + close(waiting) + } + } stopped := make(chan func(), 1) go func() { stopped <- stopBetweenApplies(say, time.Hour) }() + // Said before it blocks, and it cannot end before the apply lets go: so it has not ended now. + <-waiting select { case release := <-stopped: release() applying.Unlock() t.Fatal("the stop did not wait for the apply in flight") - case <-time.After(50 * time.Millisecond): + default: } if err := store.Save(path, store.State{Resources: []store.Applied{{ID: "nats.program-snapshot", Type: "archive"}}}); err != nil { t.Fatal(err) @@ -157,13 +167,17 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { release := <-stopped saved := make(chan error, 1) go func() { saved <- store.Save(path, store.State{}) }() + // A save after the stop blocks; nothing says it blocks, so this one wait is a bounded look. select { case <-saved: - t.Error("a save started after the stop") + release() + t.Fatal("a save started after the stop") case <-time.After(50 * time.Millisecond): } release() - <-saved + if err := <-saved; err != nil { + t.Fatal(err) + } mu.Lock() defer mu.Unlock() if len(said) != 2 || !strings.Contains(said[0], "waiting up to 1h0m0s") || !strings.Contains(said[1], "its state is saved") { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index fd1432e..5fe2f2a 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -610,7 +610,7 @@ func applyScoped( err = fmt.Errorf("%w: %s", errRemovalWaits, said) } default: - action, detail, err = remove(ctx, sys, orphan, run, made) + action, detail, err = remove(ctx, sys, orphan, run, made, journalDir(windows)) } if errors.Is(err, errNoRemoval) && store.IsFormer(orphan.ID) { // **A former target of a kind the host cannot remove is left in place and forgotten, @@ -1572,7 +1572,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru case *declaration.User: return applyUser(ctx, sys, res, run, previous, in.groups, in.handed) case *declaration.Archive: - return applyArchive(ctx, res, previous, in.away) + return applyArchive(ctx, res, previous, in.away, journalDir(in.windows)) case *declaration.Process: return applyProcess(ctx, res, run, changed, previous, in.away) case *declaration.Action: @@ -2483,7 +2483,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service, // // made is the units whose unit file this host wrote where there was none (meshMadeUnits). func remove(ctx context.Context, sys system.System, a store.Applied, run Runner, - made map[string]bool) (string, string, error) { + made map[string]bool, journals string) (string, string, error) { switch declaration.Type(a.Type) { case declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive: // Removal below a home follows no link an account put there either (novox/hq ADR 0266). @@ -2609,7 +2609,7 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner, case declaration.TypeArchive: // Exactly what it unpacked, and the directories the host made for it once they are empty // (novox/hq issue 162). - return removeArchive(a) + return removeArchive(a, journals) default: return "", "", fmt.Errorf("%w: a %q", errNoRemoval, a.Type) diff --git a/internal/apply/archive.go b/internal/apply/archive.go index 1ec3983..aeb8cd5 100644 --- a/internal/apply/archive.go +++ b/internal/apply/archive.go @@ -35,11 +35,14 @@ import ( // enough for a desktop theme and small enough to notice. const maxArchive = 512 << 20 -func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway) (Outcome, error) { +func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway, journals string) (Outcome, error) { out := begin(r) out.Action = "unchanged" recorded := previous.Wrote != "" - previous = withJournal(previous, r.Path) + previous, err := withJournal(journals, previous, r.Path) + if err != nil { + return out, err + } body, err := fetchMinding(ctx, r.Source, away) if err != nil { @@ -81,7 +84,7 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap } out.unpacked = &unpacked // At once, before the rest of the apply can fail or the engine stop (novox/hq issue 1585). - if err := keepJournal(r.Path, got, &unpacked); err != nil { + if err := keepJournal(journals, r.Path, got, &unpacked); err != nil { return out, fmt.Errorf("%s was unpacked, and what it unpacked could not be written down: %w", r.Path, err) } out.Action = "updated" diff --git a/internal/apply/archive_journal.go b/internal/apply/archive_journal.go index 3e49769..385135b 100644 --- a/internal/apply/archive_journal.go +++ b/internal/apply/archive_journal.go @@ -1,8 +1,13 @@ package apply import ( + "crypto/sha256" + "encoding/hex" "encoding/json" + "fmt" "os" + "path/filepath" + "strings" "github.com/novox/mesh-host/internal/store" ) @@ -12,53 +17,117 @@ import ( // The node's state is saved at the end of a module's apply, so an engine that stopped, was replaced or lost its // state between the unpack and that save held files on the machine it had no record of. The next apply then found // its own files and refused them as ones "the mesh did not put there" (issue 162), and a module could not move to -// its new build until a person removed them. So beside the directory, at once and atomically, the host keeps the -// digest it unpacked and the files and directories it placed; an apply that holds no record of the archive reads -// this instead of treating everything there as somebody's. The record in the state stays the authority: the -// journal is read only when there is none. +// its new build until a person removed them. So, at once and atomically, the host keeps the digest it unpacked and +// the files and directories it placed; an apply that holds no record of the archive reads this instead of treating +// everything there as somebody's. The record in the state stays the authority: the journal is read only when there +// is none. +// +// **In the node-engine's own state directory, readable and writable by root alone, never beside the archive.** The +// archive's directory may be writable by the module's account (a home, a data directory): a journal there could be +// forged, and a root engine would then remove, or take as its own, any path the forger named. And an entry is read +// only when it names a path inside the archive's directory (filepath.IsLocal), and a parent only when it is above it. -// journalSuffix is appended to an archive's path to name its journal. -const journalSuffix = ".unpacked.json" +// JournalDirName is the directory under the node's state directory that keeps the journals. +const JournalDirName = "unpacked" type unpackJournal struct { + Target string `json:"target"` Wrote string `json:"wrote"` Unpacked *store.Unpacked `json:"unpacked"` } -func journalPath(path string) string { return path + journalSuffix } +// journalDir is where the journals of this apply are kept: under the state directory every applier on the machine +// shares (the one its windows are kept in). Empty for a caller with no state directory (a test), which keeps none. +func journalDir(w *Windows) string { + if w == nil || w.dir == "" { + return "" + } + return filepath.Join(filepath.Dir(w.dir), JournalDirName) +} + +// journalPath names the journal of the archive at target: by the digest of its path, so no path names a file +// outside the journals' directory. +func journalPath(dir, target string) string { + sum := sha256.Sum256([]byte(filepath.Clean(target))) + return filepath.Join(dir, hex.EncodeToString(sum[:])+".json") +} // keepJournal writes down what an archive unpacked at path. -func keepJournal(path, wrote string, u *store.Unpacked) error { - raw, err := json.Marshal(unpackJournal{Wrote: wrote, Unpacked: u}) +func keepJournal(dir, path, wrote string, u *store.Unpacked) error { + if dir == "" { + return nil + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + raw, err := json.Marshal(unpackJournal{Target: filepath.Clean(path), Wrote: wrote, Unpacked: u}) if err != nil { return err } - return writeAtomicallyByPath(journalPath(path), raw, 0o600) + return writeAtomicallyByPath(journalPath(dir, path), raw, 0o600) } -// journalOf is what the journal says of path, or false when there is none or it cannot be read. -func journalOf(path string) (unpackJournal, bool) { - raw, err := os.ReadFile(journalPath(path)) +// journalOf is what the journal says of path: false when there is none, and an error when there is one that cannot +// be read or names a path outside the archive's directory, which is never believed. +func journalOf(dir, path string) (unpackJournal, bool, error) { + if dir == "" { + return unpackJournal{}, false, nil + } + raw, err := os.ReadFile(journalPath(dir, path)) if err != nil { - return unpackJournal{}, false + return unpackJournal{}, false, nil } var j unpackJournal - if json.Unmarshal(raw, &j) != nil || j.Wrote == "" || j.Unpacked == nil { - return unpackJournal{}, false + if err := json.Unmarshal(raw, &j); err != nil { + return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s is unreadable: %w", path, err) } - return j, true + if j.Wrote == "" || j.Unpacked == nil { + return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s names no digest or files", path) + } + if err := checkJournal(j, filepath.Clean(path)); err != nil { + return unpackJournal{}, false, err + } + return j, true, nil } -// withJournal is previous, or — when the state holds no record of the archive — what the journal says the host -// unpacked at path. -func withJournal(previous store.Applied, path string) store.Applied { - if previous.Wrote != "" { - return previous +// checkJournal refuses a journal of another target, an entry that is not a path inside the archive's directory, and +// a parent that is not above it. +func checkJournal(j unpackJournal, target string) error { + if j.Target != target { + return fmt.Errorf("the journal kept for %s names %q", target, j.Target) } - j, ok := journalOf(path) - if !ok { - return previous + for _, rel := range append(append([]string(nil), j.Unpacked.Files...), j.Unpacked.Dirs...) { + if !filepath.IsLocal(filepath.FromSlash(rel)) { + return fmt.Errorf("the journal of what was unpacked at %s names %q, which is not inside it: not believed", target, rel) + } + } + for _, p := range j.Unpacked.Parents { + if p = filepath.Clean(p); !filepath.IsAbs(p) || p == target || !strings.HasPrefix(target, p+string(filepath.Separator)) { + return fmt.Errorf("the journal of what was unpacked at %s names the parent %q, which is not above it: not believed", target, p) + } + } + return nil +} + +// withJournal is previous, or, when the state holds no record of the archive, what the journal says the host +// unpacked at path. A journal not believed is an error: the apply stops and says so rather than guess. +func withJournal(dir string, previous store.Applied, path string) (store.Applied, error) { + if previous.Wrote != "" { + return previous, nil + } + j, ok, err := journalOf(dir, path) + if err != nil || !ok { + return previous, err } previous.Wrote, previous.Target, previous.Unpacked = j.Wrote, path, j.Unpacked - return previous + return previous, nil +} + +// dropJournal removes the journal of the archive at target, once the archive is taken off; whether the state's +// record of it still said what it unpacked or not. +func dropJournal(dir, target string) { + if dir == "" || target == "" { + return + } + _ = os.Remove(journalPath(dir, target)) } diff --git a/internal/apply/archive_journal_test.go b/internal/apply/archive_journal_test.go deleted file mode 100644 index 0a56203..0000000 --- a/internal/apply/archive_journal_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package apply - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/store" -) - -// Defends novox/hq issue 1585: an archive's files were refused as "not put there by the mesh" when the state held -// no record of them, though the host had unpacked them itself. - -// An archive the host unpacked, whose record is gone, is its own: the next version replaces its files, drops the -// ones it no longer has, and leaves a file somebody added; a file the host never unpacked, or an unrecorded file -// with other bytes where no unpack was journaled, still refuses the archive. -func TestAnArchiveWhoseRecordIsGoneIsStillTheHosts(t *testing.T) { - rows := []struct { - name string - journaled bool // the first version was unpacked by the host - foreign string - wantRefuse bool - }{ - {"unpacked by the host, record lost", true, "", false}, - {"unpacked by the host, somebody's file beside", true, "mine.txt", false}, - {"never unpacked by the host", false, "", true}, - } - for _, row := range rows { - t.Run(row.name, func(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "tools") - first, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "one", "old": "o"}, "") - if row.journaled { - if _, _, err := Apply(context.Background(), archHost(t), declare(t, first), store.State{}, - store.OriginDeclared, noServices, nil, nil); err != nil { - t.Fatal(err) - } - } else { - os.MkdirAll(target, 0o755) - os.WriteFile(filepath.Join(target, "bin"), []byte("one"), 0o644) - } - if row.foreign != "" { - os.WriteFile(filepath.Join(target, row.foreign), []byte("x"), 0o644) - } - second, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "two"}, "") - // The state is empty: the record of the first unpack was lost. - _, _, err := Apply(context.Background(), archHost(t), declare(t, second), store.State{}, - store.OriginDeclared, noServices, nil, nil) - if row.wantRefuse { - if err == nil || !strings.Contains(err.Error(), "did not put there") { - t.Fatalf("refused? %v", err) - } - return - } - if err != nil { - t.Fatal(err) - } - if got, _ := os.ReadFile(filepath.Join(target, "bin")); string(got) != "two" { - t.Fatalf("bin is %q", got) - } - if exists(filepath.Join(target, "old")) { - t.Fatal("a file the new version no longer has is still there") - } - if row.foreign != "" && !exists(filepath.Join(target, row.foreign)) { - t.Fatal("somebody's file went") - } - }) - } -} - -// An unrecorded file already holding exactly the archive's bytes is not a collision; one with other bytes is. -func TestAnUnrecordedFileWithTheArchivesBytesIsNotACollision(t *testing.T) { - for _, row := range []struct { - name, has string - refused bool - }{{"same bytes", "theme", false}, {"other bytes", "somebody's", true}} { - t.Run(row.name, func(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "theme") - os.MkdirAll(target, 0o755) - os.WriteFile(filepath.Join(target, "p10k.zsh"), []byte(row.has), 0o644) - archive, _ := archiveDecl(t, "shell.theme", target, map[string]string{"p10k.zsh": "theme"}, "") - _, _, err := Apply(context.Background(), archHost(t), declare(t, archive), store.State{}, - store.OriginDeclared, noServices, nil, nil) - if (err != nil) != row.refused { - t.Fatalf("err = %v", err) - } - }) - } -} - -// Undeclaring an archive takes its journal with it, and a journal alone never makes a file the host's. -func TestTheJournalGoesWithTheArchive(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "theme") - archive, _ := archiveDecl(t, "shell.theme", target, map[string]string{"a": "a"}, "") - _, state, err := Apply(context.Background(), archHost(t), declare(t, archive), store.State{}, - store.OriginDeclared, noServices, nil, nil) - if err != nil { - t.Fatal(err) - } - if !exists(journalPath(target)) { - t.Fatal("no journal after the unpack") - } - if _, _, err := Apply(context.Background(), archHost(t), - declare(t, `{"id":"other","type":"file","path":"`+dir+`/other","content":"o"}`), state, - store.OriginDeclared, noServices, nil, nil); err != nil { - t.Fatal(err) - } - if exists(journalPath(target)) { - t.Fatal("the journal outlived the archive") - } -} diff --git a/internal/apply/archive_removal_test.go b/internal/apply/archive_removal_test.go index b9f7b67..19475dd 100644 --- a/internal/apply/archive_removal_test.go +++ b/internal/apply/archive_removal_test.go @@ -2,6 +2,7 @@ package apply import ( "context" + "encoding/json" "os" "path/filepath" "strings" @@ -152,6 +153,98 @@ func TestAnArchiveDoesNotWriteOverAFileItFound(t *testing.T) { if entries, _ := os.ReadDir(dir); len(entries) != 1 { t.Fatalf("%d entries beside the directory", len(entries)) } + + // Issue 1585: what the host unpacked is written down at once, in its own state directory, so an archive whose + // record in the state was lost is still the host's: the next version replaces its files, drops the ones it no + // longer has and leaves a file somebody added. A file the host never unpacked still refuses it, and so does a + // journal naming a path outside the archive's directory. The journal goes with the archive, record lost or not. + for _, row := range []struct { + name string + journaled bool // the first version was unpacked by the host + foreign string + forged func(j *unpackJournal) + refused string + }{ + {"unpacked by the host, record lost", true, "", nil, ""}, + {"unpacked by the host, somebody's file beside", true, "mine.txt", nil, ""}, + {"never unpacked by the host", false, "", nil, "did not put there"}, + {"a journal naming a path outside", true, "", func(j *unpackJournal) { + j.Unpacked.Files = append(j.Unpacked.Files, "../../etc/passwd") + }, "not inside it"}, + {"a journal naming a parent not above", true, "", func(j *unpackJournal) { + j.Unpacked.Parents = []string{"/etc"} + }, "not above it"}, + } { + t.Run(row.name, func(t *testing.T) { + dir, state := t.TempDir(), t.TempDir() + windows, journals := WindowsIn(state), filepath.Join(state, JournalDirName) + applyIt := func(decl string, known store.State) (store.State, error) { + _, after, err := ApplyMindingWindows(context.Background(), archHost(t), declare(t, decl), known, + store.OriginDeclared, noServices, nil, nil, nil, windows) + return after, err + } + target := filepath.Join(dir, "tools") + first, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "one", "old": "o"}, "") + if row.journaled { + if _, err := applyIt(first, store.State{}); err != nil { + t.Fatal(err) + } + if entries, _ := os.ReadDir(dir); len(entries) != 1 { + t.Fatalf("%d entries beside the archive: the journal is the state directory's", len(entries)) + } + info, err := os.Stat(journalPath(journals, target)) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("no journal of root's alone after the unpack: %v %v", info, err) + } + } else { + os.MkdirAll(target, 0o755) + os.WriteFile(filepath.Join(target, "bin"), []byte("one"), 0o644) + } + if row.forged != nil { + j, _, err := journalOf(journals, target) + if err != nil { + t.Fatal(err) + } + row.forged(&j) + raw, _ := json.Marshal(j) + os.WriteFile(journalPath(journals, target), raw, 0o600) + } + if row.foreign != "" { + os.WriteFile(filepath.Join(target, row.foreign), []byte("x"), 0o644) + } + second, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "two"}, "") + // The state is empty: the record of the first unpack was lost. + after, err := applyIt(second, store.State{}) + if row.refused != "" { + if err == nil || !strings.Contains(err.Error(), row.refused) { + t.Fatalf("refused? %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(target, "bin")); string(got) != "two" { + t.Fatalf("bin is %q", got) + } + if exists(filepath.Join(target, "old")) { + t.Fatal("a file the new version no longer has is still there") + } + if row.foreign != "" && !exists(filepath.Join(target, row.foreign)) { + t.Fatal("somebody's file went") + } + // Taken off, with a record that no longer says what it unpacked: the journal goes too. + for i := range after.Resources { + after.Resources[i].Unpacked = nil + } + if _, err := applyIt(`{"id":"other","type":"file","path":"`+dir+`/other","content":"o"}`, after); err != nil { + t.Fatal(err) + } + if exists(journalPath(journals, target)) { + t.Fatal("the journal outlived the archive") + } + }) + } } func TestADirectoryTheHostMadeGoesOnlyWhenEmpty(t *testing.T) { diff --git a/internal/apply/archive_replace_test.go b/internal/apply/archive_replace_test.go index 8b3431b..2982f1f 100644 --- a/internal/apply/archive_replace_test.go +++ b/internal/apply/archive_replace_test.go @@ -3,7 +3,6 @@ package apply import ( "context" "os" - "strings" "testing" "github.com/novox/mesh-host/internal/store" @@ -33,7 +32,7 @@ func TestAnArchiveReplacesTheTreeItWasUnpackedOver(t *testing.T) { if _, err := os.Stat(dir + "/tools/node_modules"); err == nil { t.Fatal("the previous archive's directory is still there") } - entries := besideTheTree(dir) + entries, _ := os.ReadDir(dir) if len(entries) != 1 { names := []string{} for _, e := range entries { @@ -63,19 +62,7 @@ func TestARefusedArchiveLeavesTheTreeItWouldHaveReplaced(t *testing.T) { if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "old" { t.Fatalf("index.js is %q after a refused archive", got) } - if entries := besideTheTree(dir); len(entries) != 1 { + if entries, _ := os.ReadDir(dir); len(entries) != 1 { t.Fatalf("%d entries beside the tree after a refused archive", len(entries)) } } - -// besideTheTree is what sits in dir, but for the journals of what archives unpacked (novox/hq issue 1585). -func besideTheTree(dir string) []os.DirEntry { - all, _ := os.ReadDir(dir) - var out []os.DirEntry - for _, e := range all { - if !strings.HasSuffix(e.Name(), journalSuffix) { - out = append(out, e) - } - } - return out -} diff --git a/internal/apply/archive_unpacked.go b/internal/apply/archive_unpacked.go index 5038b67..6cbce37 100644 --- a/internal/apply/archive_unpacked.go +++ b/internal/apply/archive_unpacked.go @@ -364,12 +364,13 @@ func mergeInto(fresh, path, owner string, files, dirs []string, o ours) (store.U // module's tools from its bundle — sees the whole tree or none of it, never half, and a file it has // open stays readable until it closes it. The runtime is told the module went by its own membership, // not by the files disappearing. -func removeArchive(a store.Applied) (string, string, error) { - // First, so the parents the host made are empty by the time they are looked at. - if !store.IsFormer(a.ID) && a.Unpacked != nil { - os.Remove(journalPath(filepath.Clean(a.Target))) +func removeArchive(a store.Applied, journals string) (string, string, error) { + action, detail, err := removeArchiveFiles(a) + // And its journal (issue 1585), once the record goes: whether the record said what it unpacked or not. + if err == nil { + dropJournal(journals, a.Target) } - return removeArchiveFiles(a) + return action, detail, err } func removeArchiveFiles(a store.Applied) (string, string, error) { diff --git a/internal/apply/home_links_test.go b/internal/apply/home_links_test.go index 97a47e3..aad8d10 100644 --- a/internal/apply/home_links_test.go +++ b/internal/apply/home_links_test.go @@ -159,7 +159,7 @@ func TestRemovalThroughALinkUnderAHomeIsRefused(t *testing.T) { home, outside := aLinkedHome(t) link(t, outside, filepath.Join(home, ".claude")) _, _, err := remove(context.Background(), archHost(t), store.Applied{ID: "m.f", Type: "file", - Target: filepath.Join(home, ".claude", "shadow")}, nil, nil) + Target: filepath.Join(home, ".claude", "shadow")}, nil, nil, "") mustBeLinkRefusal(t, err) if _, err := os.Stat(filepath.Join(outside, "shadow")); err != nil { t.Fatal("removed through the link") diff --git a/module.json b/module.json index 57270a3..b71573b 100644 --- a/module.json +++ b/module.json @@ -49,7 +49,7 @@ "type": "file", "path": "/usr/lib/nox-mesh-host/launch", "mode": "0755", - "content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A\n# delivered host that is not the known-good one runs on trial: it must report its declaration\n# under its own build — which it marks by writing itself into known-good — within a bound. One\n# that crashes repeatedly, exits without standing aside for anything, or does not report in\n# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host\n# says every record on its reports, so the mesh raises it as a condition. One rollback per\n# version: a version in `rolled-back` is never started again; a newer delivery is.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything — which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares\n# no code with the host and calls none of it: a binary that cannot start cannot be its own\n# recovery.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\n# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.\nBOUND=\"${MESH_HOST_TRIAL_BOUND:-600}\"\nTICK=\"${MESH_HOST_TRIAL_TICK:-5}\"\nGRACE=\"${MESH_HOST_STOP_GRACE:-20}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\nKNOWN_GOOD=\"$STATE_DIR/known-good\"\n# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read\n# by the host (internal/upgrade) and said on its reports.\nROLLED=\"$STATE_DIR/rolled-back\"\nTRIAL=\"$STATE_DIR/trial\"\nEXPIRED=\"$STATE_DIR/trial-expired\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\nnow() { date +%s; }\n\nknown_good() { tr -d '[:space:]' < \"$KNOWN_GOOD\" 2>/dev/null || true; }\n\ndelivered() { [ -n \"$1\" ] && [ -x \"$VERSIONS/$1/$BINARY\" ]; }\n\nrolled_back() { [ -s \"$ROLLED\" ] && cut -f1 \"$ROLLED\" 2>/dev/null | grep -qxF \"$1\"; }\n\n# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.\nversion_of() {\n\tcase \"$1\" in\n\t\t\"$VERSIONS\"/*/\"$BINARY\") v=\"${1#\"$VERSIONS\"/}\"; echo \"${v%/\"$BINARY\"}\" ;;\n\t\t*) echo \"\" ;;\n\tesac\n}\n\nrecord() { # from to outcome why\n\tprintf '%s\\t%s\\t%s\\t%s\\t%s\\n' \"$1\" \"$2\" \"$(now)\" \"$3\" \"$4\" >> \"$ROLLED\"\n}\n\n# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin\n# stands, the version it pinned — or the one placed by hand when nothing has been delivered.\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW — resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort — \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\n#\n# **A pin stands until something newer arrives.** A version delivered after the pin was written, and\n# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.\npick_host() {\n\tnewest=\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tdelivered \"$candidate\" || continue\n\t\trolled_back \"$candidate\" && continue\n\t\tnewest=\"$candidate\"\n\t\tbreak\n\tdone\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$newest\" ] && [ \"$newest\" != \"$pinned\" ] && [ \"$VERSIONS/$newest\" -nt \"$PINNED\" ]; then\n\t\t\tsay \"host $newest arrived after the rollback to $pinned; running it\"\n\t\t\trm -f \"$PINNED\"\n\t\telif delivered \"$pinned\"; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\telse\n\t\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\t\tfi\n\tfi\n\tif [ -n \"$newest\" ]; then\n\t\techo \"$VERSIONS/$newest/$BINARY\"\n\t\treturn 0\n\tfi\n\techo \"$FALLBACK\"\n}\n\n# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.\nroll_back() { # from why\n\tkg=\"$(known_good)\"\n\tif [ -z \"$1\" ] || [ \"$1\" = \"$kg\" ] || ! delivered \"$kg\"; then\n\t\treturn 1\n\tfi\n\trecord \"$1\" \"$kg\" rolled-back \"$2\"\n\t# The pin is what stops the launcher starting a version newer than known-good that is not the one\n\t# rolled back; the record is what stops it starting this one again.\n\tprintf '%s\\n' \"$kg\" > \"$PINNED\"\n\trm -f \"$TRIAL\"\n\tsay \"rolled back from host $1 to $kg: $2\"\n\treturn 0\n}\n\n# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.\n# A subshell beside the host, so the launcher's own wait is untouched.\ntrial_watch() { # pid version deadline\n\twhile kill -0 \"$1\" 2>/dev/null; do\n\t\t[ \"$(known_good)\" = \"$2\" ] && return 0\n\t\tif [ \"$(now)\" -ge \"$3\" ]; then\n\t\t\tprintf '%s\\n' \"$2\" > \"$EXPIRED\"\n\t\t\tsay \"host $2 did not report within ${BOUND}s of starting; stopping it\"\n\t\t\tkill -TERM \"$1\" 2>/dev/null\n\t\t\twaited=0\n\t\t\twhile kill -0 \"$1\" 2>/dev/null && [ \"$waited\" -lt \"$GRACE\" ]; do\n\t\t\t\tsleep 1\n\t\t\t\twaited=$((waited + 1))\n\t\t\tdone\n\t\t\tkill -KILL \"$1\" 2>/dev/null\n\t\t\treturn 0\n\t\tfi\n\t\tsleep \"$TICK\"\n\tdone\n}\n\nchild=\nwatcher=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at\n# the next boot.\nSELF=\"$(cksum < \"$0\" 2>/dev/null || true)\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is\n\t# the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tHOST=\"$(pick_host)\"\n\tversion=\"$(version_of \"$HOST\")\"\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif roll_back \"$version\" \"it failed $count times in a row\"; then\n\t\t\t# Fresh count for the version now run: it deserves its own attempts, and without this\n\t\t\t# it inherits a count already over the limit and halts at once. The variable too, not\n\t\t\t# only the file: resetting one and not the other made the next failure count from the\n\t\t\t# OLD value — so the rolled-back version got one attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\t\tcontinue\n\t\tfi\n\t\tkg=\"$(known_good)\"\n\t\tif [ -n \"$kg\" ] && { [ \"$version\" = \"$kg\" ] || [ -z \"$version\" ]; } && [ -s \"$ROLLED\" ]; then\n\t\t\t# Already gone back, and what it went back to fails as well: this is the machine and\n\t\t\t# not a binary. Rolling back again would flap between two versions for ever.\n\t\t\tsay \"the host failed $count times after a rollback. the version it went back to does\"\n\t\t\tsay \"not start either, so this is the machine and not the binary.\"\n\t\t\trecord \"${version:-placed-by-hand}\" \"\" halted \"the version rolled back to failed $count times as well\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\t\t# Nothing to go back to: no host has ever reported here, or the one that did was placed by\n\t\t# hand and is not delivered. An installation failure rather than an upgrade's — said, and\n\t\t# tried again slowly, never guessed at.\n\t\tsay \"the host failed $count times and there is no delivered known-good version to go back to.\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\tfi\n\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\n\t# **On trial**: a delivered version that is not the known-good one, while a known-good one is\n\t# delivered to go back to. The trial starts when this version was first started, and survives\n\t# this launcher restarting.\n\ttrial=\n\tdeadline=\n\tkg=\"$(known_good)\"\n\tif [ -n \"$version\" ] && [ \"$version\" != \"$kg\" ] && delivered \"$kg\"; then\n\t\ttrial=yes\n\t\tstarted=\n\t\tif [ -s \"$TRIAL\" ]; then\n\t\t\tread -r on since _ < \"$TRIAL\" 2>/dev/null || on=\n\t\t\t[ \"${on:-}\" = \"$version\" ] && started=\"${since:-}\"\n\t\tfi\n\t\tcase \"$started\" in\n\t\t\t'' | *[!0-9]*) started=\"$(now)\"; printf '%s %s\\n' \"$version\" \"$started\" > \"$TRIAL\" ;;\n\t\tesac\n\t\tdeadline=$((started + BOUND))\n\t\tif [ \"$(now)\" -ge \"$deadline\" ]; then\n\t\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\" && continue\n\t\tfi\n\telse\n\t\trm -f \"$TRIAL\"\n\tfi\n\n\trm -f \"$EXPIRED\"\n\tsay \"running $HOST${trial:+ (on trial until it reports)}\"\n\t\"$HOST\" run &\n\tchild=$!\n\twatcher=\n\tif [ -n \"$trial\" ]; then\n\t\ttrial_watch \"$child\" \"$version\" \"$deadline\" &\n\t\twatcher=$!\n\tfi\n\tstatus=0\n\twait \"$child\" || status=$?\n\tif [ -n \"$stopping\" ]; then\n\t\t# The signal interrupted the wait, not the host: it was told, and is finishing what it was\n\t\t# doing. Waited for, so the service manager does not kill it half way through an apply.\n\t\twait \"$child\" 2>/dev/null\n\tfi\n\tchild=\n\tif [ -n \"$watcher\" ]; then\n\t\tkill \"$watcher\" 2>/dev/null\n\t\twait \"$watcher\" 2>/dev/null\n\t\twatcher=\n\tfi\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -n \"$trial\" ] && [ -s \"$EXPIRED\" ] && [ \"$(cat \"$EXPIRED\" 2>/dev/null)\" = \"$version\" ]; then\n\t\trm -f \"$EXPIRED\"\n\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\tcontinue\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\t#\n\t\t\t# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the\n\t\t\t# same host is still the one to run has not stood aside for a successor: it has\n\t\t\t# stopped, and a host that stops at once would otherwise loop here for ever unseen.\n\t\t\tif [ -n \"$trial\" ] && [ \"$(pick_host)\" = \"$HOST\" ] && [ \"$(known_good)\" != \"$version\" ]; then\n\t\t\t\tsay \"host $version exited cleanly on trial with nothing newer to stand aside for\"\n\t\t\telse\n\t\t\t\t# A delivered successor of this launcher runs from here on, not from the next boot.\n\t\t\t\tif [ -n \"$SELF\" ] && [ \"$(cksum < \"$0\" 2>/dev/null || true)\" != \"$SELF\" ]; then\n\t\t\t\t\tsay \"the launcher was replaced; running the new one\"\n\t\t\t\t\texec \"$0\"\n\t\t\t\tfi\n\t\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\t\tcontinue\n\t\t\tfi\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n" + "content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A\n# delivered host that is not the known-good one runs on trial: it must report its declaration\n# under its own build — which it marks by writing itself into known-good — within a bound. One\n# that crashes repeatedly, exits without standing aside for anything, or does not report in\n# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host\n# says every record on its reports, so the mesh raises it as a condition. One rollback per\n# version: a version in `rolled-back` is never started again; a newer delivery is.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything — which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares\n# no code with the host and calls none of it: a binary that cannot start cannot be its own\n# recovery.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\n# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.\nBOUND=\"${MESH_HOST_TRIAL_BOUND:-600}\"\nTICK=\"${MESH_HOST_TRIAL_TICK:-5}\"\n# How long a host told to stop is waited for before it is killed, in seconds: past the 60 s the host itself waits for\n# the apply in flight to end and save (novox/hq issue 1585), so a stop never cuts a save off.\nGRACE=\"${MESH_HOST_STOP_GRACE:-75}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\nKNOWN_GOOD=\"$STATE_DIR/known-good\"\n# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read\n# by the host (internal/upgrade) and said on its reports.\nROLLED=\"$STATE_DIR/rolled-back\"\nTRIAL=\"$STATE_DIR/trial\"\nEXPIRED=\"$STATE_DIR/trial-expired\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\nnow() { date +%s; }\n\nknown_good() { tr -d '[:space:]' < \"$KNOWN_GOOD\" 2>/dev/null || true; }\n\ndelivered() { [ -n \"$1\" ] && [ -x \"$VERSIONS/$1/$BINARY\" ]; }\n\nrolled_back() { [ -s \"$ROLLED\" ] && cut -f1 \"$ROLLED\" 2>/dev/null | grep -qxF \"$1\"; }\n\n# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.\nversion_of() {\n\tcase \"$1\" in\n\t\t\"$VERSIONS\"/*/\"$BINARY\") v=\"${1#\"$VERSIONS\"/}\"; echo \"${v%/\"$BINARY\"}\" ;;\n\t\t*) echo \"\" ;;\n\tesac\n}\n\nrecord() { # from to outcome why\n\tprintf '%s\\t%s\\t%s\\t%s\\t%s\\n' \"$1\" \"$2\" \"$(now)\" \"$3\" \"$4\" >> \"$ROLLED\"\n}\n\n# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin\n# stands, the version it pinned — or the one placed by hand when nothing has been delivered.\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW — resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort — \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\n#\n# **A pin stands until something newer arrives.** A version delivered after the pin was written, and\n# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.\npick_host() {\n\tnewest=\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tdelivered \"$candidate\" || continue\n\t\trolled_back \"$candidate\" && continue\n\t\tnewest=\"$candidate\"\n\t\tbreak\n\tdone\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$newest\" ] && [ \"$newest\" != \"$pinned\" ] && [ \"$VERSIONS/$newest\" -nt \"$PINNED\" ]; then\n\t\t\tsay \"host $newest arrived after the rollback to $pinned; running it\"\n\t\t\trm -f \"$PINNED\"\n\t\telif delivered \"$pinned\"; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\telse\n\t\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\t\tfi\n\tfi\n\tif [ -n \"$newest\" ]; then\n\t\techo \"$VERSIONS/$newest/$BINARY\"\n\t\treturn 0\n\tfi\n\techo \"$FALLBACK\"\n}\n\n# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.\nroll_back() { # from why\n\tkg=\"$(known_good)\"\n\tif [ -z \"$1\" ] || [ \"$1\" = \"$kg\" ] || ! delivered \"$kg\"; then\n\t\treturn 1\n\tfi\n\trecord \"$1\" \"$kg\" rolled-back \"$2\"\n\t# The pin is what stops the launcher starting a version newer than known-good that is not the one\n\t# rolled back; the record is what stops it starting this one again.\n\tprintf '%s\\n' \"$kg\" > \"$PINNED\"\n\trm -f \"$TRIAL\"\n\tsay \"rolled back from host $1 to $kg: $2\"\n\treturn 0\n}\n\n# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.\n# A subshell beside the host, so the launcher's own wait is untouched.\ntrial_watch() { # pid version deadline\n\twhile kill -0 \"$1\" 2>/dev/null; do\n\t\t[ \"$(known_good)\" = \"$2\" ] && return 0\n\t\tif [ \"$(now)\" -ge \"$3\" ]; then\n\t\t\tprintf '%s\\n' \"$2\" > \"$EXPIRED\"\n\t\t\tsay \"host $2 did not report within ${BOUND}s of starting; stopping it\"\n\t\t\tkill -TERM \"$1\" 2>/dev/null\n\t\t\twaited=0\n\t\t\twhile kill -0 \"$1\" 2>/dev/null && [ \"$waited\" -lt \"$GRACE\" ]; do\n\t\t\t\tsleep 1\n\t\t\t\twaited=$((waited + 1))\n\t\t\tdone\n\t\t\tkill -KILL \"$1\" 2>/dev/null\n\t\t\treturn 0\n\t\tfi\n\t\tsleep \"$TICK\"\n\tdone\n}\n\nchild=\nwatcher=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at\n# the next boot.\nSELF=\"$(cksum < \"$0\" 2>/dev/null || true)\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is\n\t# the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tHOST=\"$(pick_host)\"\n\tversion=\"$(version_of \"$HOST\")\"\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif roll_back \"$version\" \"it failed $count times in a row\"; then\n\t\t\t# Fresh count for the version now run: it deserves its own attempts, and without this\n\t\t\t# it inherits a count already over the limit and halts at once. The variable too, not\n\t\t\t# only the file: resetting one and not the other made the next failure count from the\n\t\t\t# OLD value — so the rolled-back version got one attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\t\tcontinue\n\t\tfi\n\t\tkg=\"$(known_good)\"\n\t\tif [ -n \"$kg\" ] && { [ \"$version\" = \"$kg\" ] || [ -z \"$version\" ]; } && [ -s \"$ROLLED\" ]; then\n\t\t\t# Already gone back, and what it went back to fails as well: this is the machine and\n\t\t\t# not a binary. Rolling back again would flap between two versions for ever.\n\t\t\tsay \"the host failed $count times after a rollback. the version it went back to does\"\n\t\t\tsay \"not start either, so this is the machine and not the binary.\"\n\t\t\trecord \"${version:-placed-by-hand}\" \"\" halted \"the version rolled back to failed $count times as well\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\t\t# Nothing to go back to: no host has ever reported here, or the one that did was placed by\n\t\t# hand and is not delivered. An installation failure rather than an upgrade's — said, and\n\t\t# tried again slowly, never guessed at.\n\t\tsay \"the host failed $count times and there is no delivered known-good version to go back to.\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\tfi\n\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\n\t# **On trial**: a delivered version that is not the known-good one, while a known-good one is\n\t# delivered to go back to. The trial starts when this version was first started, and survives\n\t# this launcher restarting.\n\ttrial=\n\tdeadline=\n\tkg=\"$(known_good)\"\n\tif [ -n \"$version\" ] && [ \"$version\" != \"$kg\" ] && delivered \"$kg\"; then\n\t\ttrial=yes\n\t\tstarted=\n\t\tif [ -s \"$TRIAL\" ]; then\n\t\t\tread -r on since _ < \"$TRIAL\" 2>/dev/null || on=\n\t\t\t[ \"${on:-}\" = \"$version\" ] && started=\"${since:-}\"\n\t\tfi\n\t\tcase \"$started\" in\n\t\t\t'' | *[!0-9]*) started=\"$(now)\"; printf '%s %s\\n' \"$version\" \"$started\" > \"$TRIAL\" ;;\n\t\tesac\n\t\tdeadline=$((started + BOUND))\n\t\tif [ \"$(now)\" -ge \"$deadline\" ]; then\n\t\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\" && continue\n\t\tfi\n\telse\n\t\trm -f \"$TRIAL\"\n\tfi\n\n\trm -f \"$EXPIRED\"\n\tsay \"running $HOST${trial:+ (on trial until it reports)}\"\n\t\"$HOST\" run &\n\tchild=$!\n\twatcher=\n\tif [ -n \"$trial\" ]; then\n\t\ttrial_watch \"$child\" \"$version\" \"$deadline\" &\n\t\twatcher=$!\n\tfi\n\tstatus=0\n\twait \"$child\" || status=$?\n\tif [ -n \"$stopping\" ]; then\n\t\t# The signal interrupted the wait, not the host: it was told, and is finishing what it was\n\t\t# doing. Waited for, so the service manager does not kill it half way through an apply.\n\t\twait \"$child\" 2>/dev/null\n\tfi\n\tchild=\n\tif [ -n \"$watcher\" ]; then\n\t\tkill \"$watcher\" 2>/dev/null\n\t\twait \"$watcher\" 2>/dev/null\n\t\twatcher=\n\tfi\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -n \"$trial\" ] && [ -s \"$EXPIRED\" ] && [ \"$(cat \"$EXPIRED\" 2>/dev/null)\" = \"$version\" ]; then\n\t\trm -f \"$EXPIRED\"\n\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\tcontinue\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\t#\n\t\t\t# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the\n\t\t\t# same host is still the one to run has not stood aside for a successor: it has\n\t\t\t# stopped, and a host that stops at once would otherwise loop here for ever unseen.\n\t\t\tif [ -n \"$trial\" ] && [ \"$(pick_host)\" = \"$HOST\" ] && [ \"$(known_good)\" != \"$version\" ]; then\n\t\t\t\tsay \"host $version exited cleanly on trial with nothing newer to stand aside for\"\n\t\t\telse\n\t\t\t\t# A delivered successor of this launcher runs from here on, not from the next boot.\n\t\t\t\tif [ -n \"$SELF\" ] && [ \"$(cksum < \"$0\" 2>/dev/null || true)\" != \"$SELF\" ]; then\n\t\t\t\t\tsay \"the launcher was replaced; running the new one\"\n\t\t\t\t\texec \"$0\"\n\t\t\t\tfi\n\t\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\t\tcontinue\n\t\t\tfi\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n" }, { "id": "next", diff --git a/packaging/nox-mesh-host-launch b/packaging/nox-mesh-host-launch index e75d59a..cf946a2 100755 --- a/packaging/nox-mesh-host-launch +++ b/packaging/nox-mesh-host-launch @@ -38,7 +38,9 @@ BACKOFF="${MESH_HOST_BACKOFF:-5}" # How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply. BOUND="${MESH_HOST_TRIAL_BOUND:-600}" TICK="${MESH_HOST_TRIAL_TICK:-5}" -GRACE="${MESH_HOST_STOP_GRACE:-20}" +# How long a host told to stop is waited for before it is killed, in seconds: past the 60 s the host itself waits for +# the apply in flight to end and save (novox/hq issue 1585), so a stop never cuts a save off. +GRACE="${MESH_HOST_STOP_GRACE:-75}" ONCE="${MESH_HOST_RUN_ONCE:-}" # tests run one iteration; nothing else sets this ATTEMPTS="$STATE_DIR/start-attempts" diff --git a/packaging/nox-mesh-host.openrc b/packaging/nox-mesh-host.openrc index cc740f0..7e96b88 100644 --- a/packaging/nox-mesh-host.openrc +++ b/packaging/nox-mesh-host.openrc @@ -5,4 +5,6 @@ name="nox-mesh-host" command="/usr/lib/nox-mesh-host/launch" supervisor="supervise-daemon" +# The same stop bound as the systemd unit's TimeoutStopSec (issue 1585): TERM, 120 s, then KILL. +retry="TERM/120/KILL/5" depend() { need net; } diff --git a/packaging/nox-mesh-host.service b/packaging/nox-mesh-host.service index c9c6df1..05e97dc 100644 --- a/packaging/nox-mesh-host.service +++ b/packaging/nox-mesh-host.service @@ -12,6 +12,9 @@ Restart=always RestartSec=5s StateDirectory=mesh-host KillMode=mixed +# Past the launcher's own stop grace (75 s), which is past the 60 s the host waits for the apply in +# flight to end and save (novox/hq issue 1585): the service manager's kill is the last resort. +TimeoutStopSec=120 [Install] WantedBy=multi-user.target