diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 72a43f6..cd4df78 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -140,15 +140,25 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { applying.Lock() var said []string var mu sync.Mutex - say := func(line string) { mu.Lock(); said = append(said, line); mu.Unlock() } + waiting := make(chan struct{}) + say := func(line string) { + mu.Lock() + said = append(said, line) + mu.Unlock() + if strings.Contains(line, "waiting up to") { + close(waiting) + } + } stopped := make(chan func(), 1) go func() { stopped <- stopBetweenApplies(say, time.Hour) }() + // Said before it blocks, and it cannot end before the apply lets go: so it has not ended now. + <-waiting select { case release := <-stopped: release() applying.Unlock() t.Fatal("the stop did not wait for the apply in flight") - case <-time.After(50 * time.Millisecond): + default: } if err := store.Save(path, store.State{Resources: []store.Applied{{ID: "nats.program-snapshot", Type: "archive"}}}); err != nil { t.Fatal(err) @@ -157,13 +167,17 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { release := <-stopped saved := make(chan error, 1) go func() { saved <- store.Save(path, store.State{}) }() + // A save after the stop blocks; nothing says it blocks, so this one wait is a bounded look. select { case <-saved: - t.Error("a save started after the stop") + release() + t.Fatal("a save started after the stop") case <-time.After(50 * time.Millisecond): } release() - <-saved + if err := <-saved; err != nil { + t.Fatal(err) + } mu.Lock() defer mu.Unlock() if len(said) != 2 || !strings.Contains(said[0], "waiting up to 1h0m0s") || !strings.Contains(said[1], "its state is saved") { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index fd1432e..5fe2f2a 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -610,7 +610,7 @@ func applyScoped( err = fmt.Errorf("%w: %s", errRemovalWaits, said) } default: - action, detail, err = remove(ctx, sys, orphan, run, made) + action, detail, err = remove(ctx, sys, orphan, run, made, journalDir(windows)) } if errors.Is(err, errNoRemoval) && store.IsFormer(orphan.ID) { // **A former target of a kind the host cannot remove is left in place and forgotten, @@ -1572,7 +1572,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru case *declaration.User: return applyUser(ctx, sys, res, run, previous, in.groups, in.handed) case *declaration.Archive: - return applyArchive(ctx, res, previous, in.away) + return applyArchive(ctx, res, previous, in.away, journalDir(in.windows)) case *declaration.Process: return applyProcess(ctx, res, run, changed, previous, in.away) case *declaration.Action: @@ -2483,7 +2483,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service, // // made is the units whose unit file this host wrote where there was none (meshMadeUnits). func remove(ctx context.Context, sys system.System, a store.Applied, run Runner, - made map[string]bool) (string, string, error) { + made map[string]bool, journals string) (string, string, error) { switch declaration.Type(a.Type) { case declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive: // Removal below a home follows no link an account put there either (novox/hq ADR 0266). @@ -2609,7 +2609,7 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner, case declaration.TypeArchive: // Exactly what it unpacked, and the directories the host made for it once they are empty // (novox/hq issue 162). - return removeArchive(a) + return removeArchive(a, journals) default: return "", "", fmt.Errorf("%w: a %q", errNoRemoval, a.Type) diff --git a/internal/apply/archive.go b/internal/apply/archive.go index 1ec3983..aeb8cd5 100644 --- a/internal/apply/archive.go +++ b/internal/apply/archive.go @@ -35,11 +35,14 @@ import ( // enough for a desktop theme and small enough to notice. const maxArchive = 512 << 20 -func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway) (Outcome, error) { +func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway, journals string) (Outcome, error) { out := begin(r) out.Action = "unchanged" recorded := previous.Wrote != "" - previous = withJournal(previous, r.Path) + previous, err := withJournal(journals, previous, r.Path) + if err != nil { + return out, err + } body, err := fetchMinding(ctx, r.Source, away) if err != nil { @@ -81,7 +84,7 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap } out.unpacked = &unpacked // At once, before the rest of the apply can fail or the engine stop (novox/hq issue 1585). - if err := keepJournal(r.Path, got, &unpacked); err != nil { + if err := keepJournal(journals, r.Path, got, &unpacked); err != nil { return out, fmt.Errorf("%s was unpacked, and what it unpacked could not be written down: %w", r.Path, err) } out.Action = "updated" diff --git a/internal/apply/archive_journal.go b/internal/apply/archive_journal.go index 3e49769..385135b 100644 --- a/internal/apply/archive_journal.go +++ b/internal/apply/archive_journal.go @@ -1,8 +1,13 @@ package apply import ( + "crypto/sha256" + "encoding/hex" "encoding/json" + "fmt" "os" + "path/filepath" + "strings" "github.com/novox/mesh-host/internal/store" ) @@ -12,53 +17,117 @@ import ( // The node's state is saved at the end of a module's apply, so an engine that stopped, was replaced or lost its // state between the unpack and that save held files on the machine it had no record of. The next apply then found // its own files and refused them as ones "the mesh did not put there" (issue 162), and a module could not move to -// its new build until a person removed them. So beside the directory, at once and atomically, the host keeps the -// digest it unpacked and the files and directories it placed; an apply that holds no record of the archive reads -// this instead of treating everything there as somebody's. The record in the state stays the authority: the -// journal is read only when there is none. +// its new build until a person removed them. So, at once and atomically, the host keeps the digest it unpacked and +// the files and directories it placed; an apply that holds no record of the archive reads this instead of treating +// everything there as somebody's. The record in the state stays the authority: the journal is read only when there +// is none. +// +// **In the node-engine's own state directory, readable and writable by root alone, never beside the archive.** The +// archive's directory may be writable by the module's account (a home, a data directory): a journal there could be +// forged, and a root engine would then remove, or take as its own, any path the forger named. And an entry is read +// only when it names a path inside the archive's directory (filepath.IsLocal), and a parent only when it is above it. -// journalSuffix is appended to an archive's path to name its journal. -const journalSuffix = ".unpacked.json" +// JournalDirName is the directory under the node's state directory that keeps the journals. +const JournalDirName = "unpacked" type unpackJournal struct { + Target string `json:"target"` Wrote string `json:"wrote"` Unpacked *store.Unpacked `json:"unpacked"` } -func journalPath(path string) string { return path + journalSuffix } +// journalDir is where the journals of this apply are kept: under the state directory every applier on the machine +// shares (the one its windows are kept in). Empty for a caller with no state directory (a test), which keeps none. +func journalDir(w *Windows) string { + if w == nil || w.dir == "" { + return "" + } + return filepath.Join(filepath.Dir(w.dir), JournalDirName) +} + +// journalPath names the journal of the archive at target: by the digest of its path, so no path names a file +// outside the journals' directory. +func journalPath(dir, target string) string { + sum := sha256.Sum256([]byte(filepath.Clean(target))) + return filepath.Join(dir, hex.EncodeToString(sum[:])+".json") +} // keepJournal writes down what an archive unpacked at path. -func keepJournal(path, wrote string, u *store.Unpacked) error { - raw, err := json.Marshal(unpackJournal{Wrote: wrote, Unpacked: u}) +func keepJournal(dir, path, wrote string, u *store.Unpacked) error { + if dir == "" { + return nil + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + raw, err := json.Marshal(unpackJournal{Target: filepath.Clean(path), Wrote: wrote, Unpacked: u}) if err != nil { return err } - return writeAtomicallyByPath(journalPath(path), raw, 0o600) + return writeAtomicallyByPath(journalPath(dir, path), raw, 0o600) } -// journalOf is what the journal says of path, or false when there is none or it cannot be read. -func journalOf(path string) (unpackJournal, bool) { - raw, err := os.ReadFile(journalPath(path)) +// journalOf is what the journal says of path: false when there is none, and an error when there is one that cannot +// be read or names a path outside the archive's directory, which is never believed. +func journalOf(dir, path string) (unpackJournal, bool, error) { + if dir == "" { + return unpackJournal{}, false, nil + } + raw, err := os.ReadFile(journalPath(dir, path)) if err != nil { - return unpackJournal{}, false + return unpackJournal{}, false, nil } var j unpackJournal - if json.Unmarshal(raw, &j) != nil || j.Wrote == "" || j.Unpacked == nil { - return unpackJournal{}, false + if err := json.Unmarshal(raw, &j); err != nil { + return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s is unreadable: %w", path, err) } - return j, true + if j.Wrote == "" || j.Unpacked == nil { + return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s names no digest or files", path) + } + if err := checkJournal(j, filepath.Clean(path)); err != nil { + return unpackJournal{}, false, err + } + return j, true, nil } -// withJournal is previous, or — when the state holds no record of the archive — what the journal says the host -// unpacked at path. -func withJournal(previous store.Applied, path string) store.Applied { - if previous.Wrote != "" { - return previous +// checkJournal refuses a journal of another target, an entry that is not a path inside the archive's directory, and +// a parent that is not above it. +func checkJournal(j unpackJournal, target string) error { + if j.Target != target { + return fmt.Errorf("the journal kept for %s names %q", target, j.Target) } - j, ok := journalOf(path) - if !ok { - return previous + for _, rel := range append(append([]string(nil), j.Unpacked.Files...), j.Unpacked.Dirs...) { + if !filepath.IsLocal(filepath.FromSlash(rel)) { + return fmt.Errorf("the journal of what was unpacked at %s names %q, which is not inside it: not believed", target, rel) + } + } + for _, p := range j.Unpacked.Parents { + if p = filepath.Clean(p); !filepath.IsAbs(p) || p == target || !strings.HasPrefix(target, p+string(filepath.Separator)) { + return fmt.Errorf("the journal of what was unpacked at %s names the parent %q, which is not above it: not believed", target, p) + } + } + return nil +} + +// withJournal is previous, or, when the state holds no record of the archive, what the journal says the host +// unpacked at path. A journal not believed is an error: the apply stops and says so rather than guess. +func withJournal(dir string, previous store.Applied, path string) (store.Applied, error) { + if previous.Wrote != "" { + return previous, nil + } + j, ok, err := journalOf(dir, path) + if err != nil || !ok { + return previous, err } previous.Wrote, previous.Target, previous.Unpacked = j.Wrote, path, j.Unpacked - return previous + return previous, nil +} + +// dropJournal removes the journal of the archive at target, once the archive is taken off; whether the state's +// record of it still said what it unpacked or not. +func dropJournal(dir, target string) { + if dir == "" || target == "" { + return + } + _ = os.Remove(journalPath(dir, target)) } diff --git a/internal/apply/archive_journal_test.go b/internal/apply/archive_journal_test.go deleted file mode 100644 index 0a56203..0000000 --- a/internal/apply/archive_journal_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package apply - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/store" -) - -// Defends novox/hq issue 1585: an archive's files were refused as "not put there by the mesh" when the state held -// no record of them, though the host had unpacked them itself. - -// An archive the host unpacked, whose record is gone, is its own: the next version replaces its files, drops the -// ones it no longer has, and leaves a file somebody added; a file the host never unpacked, or an unrecorded file -// with other bytes where no unpack was journaled, still refuses the archive. -func TestAnArchiveWhoseRecordIsGoneIsStillTheHosts(t *testing.T) { - rows := []struct { - name string - journaled bool // the first version was unpacked by the host - foreign string - wantRefuse bool - }{ - {"unpacked by the host, record lost", true, "", false}, - {"unpacked by the host, somebody's file beside", true, "mine.txt", false}, - {"never unpacked by the host", false, "", true}, - } - for _, row := range rows { - t.Run(row.name, func(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "tools") - first, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "one", "old": "o"}, "") - if row.journaled { - if _, _, err := Apply(context.Background(), archHost(t), declare(t, first), store.State{}, - store.OriginDeclared, noServices, nil, nil); err != nil { - t.Fatal(err) - } - } else { - os.MkdirAll(target, 0o755) - os.WriteFile(filepath.Join(target, "bin"), []byte("one"), 0o644) - } - if row.foreign != "" { - os.WriteFile(filepath.Join(target, row.foreign), []byte("x"), 0o644) - } - second, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "two"}, "") - // The state is empty: the record of the first unpack was lost. - _, _, err := Apply(context.Background(), archHost(t), declare(t, second), store.State{}, - store.OriginDeclared, noServices, nil, nil) - if row.wantRefuse { - if err == nil || !strings.Contains(err.Error(), "did not put there") { - t.Fatalf("refused? %v", err) - } - return - } - if err != nil { - t.Fatal(err) - } - if got, _ := os.ReadFile(filepath.Join(target, "bin")); string(got) != "two" { - t.Fatalf("bin is %q", got) - } - if exists(filepath.Join(target, "old")) { - t.Fatal("a file the new version no longer has is still there") - } - if row.foreign != "" && !exists(filepath.Join(target, row.foreign)) { - t.Fatal("somebody's file went") - } - }) - } -} - -// An unrecorded file already holding exactly the archive's bytes is not a collision; one with other bytes is. -func TestAnUnrecordedFileWithTheArchivesBytesIsNotACollision(t *testing.T) { - for _, row := range []struct { - name, has string - refused bool - }{{"same bytes", "theme", false}, {"other bytes", "somebody's", true}} { - t.Run(row.name, func(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "theme") - os.MkdirAll(target, 0o755) - os.WriteFile(filepath.Join(target, "p10k.zsh"), []byte(row.has), 0o644) - archive, _ := archiveDecl(t, "shell.theme", target, map[string]string{"p10k.zsh": "theme"}, "") - _, _, err := Apply(context.Background(), archHost(t), declare(t, archive), store.State{}, - store.OriginDeclared, noServices, nil, nil) - if (err != nil) != row.refused { - t.Fatalf("err = %v", err) - } - }) - } -} - -// Undeclaring an archive takes its journal with it, and a journal alone never makes a file the host's. -func TestTheJournalGoesWithTheArchive(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "theme") - archive, _ := archiveDecl(t, "shell.theme", target, map[string]string{"a": "a"}, "") - _, state, err := Apply(context.Background(), archHost(t), declare(t, archive), store.State{}, - store.OriginDeclared, noServices, nil, nil) - if err != nil { - t.Fatal(err) - } - if !exists(journalPath(target)) { - t.Fatal("no journal after the unpack") - } - if _, _, err := Apply(context.Background(), archHost(t), - declare(t, `{"id":"other","type":"file","path":"`+dir+`/other","content":"o"}`), state, - store.OriginDeclared, noServices, nil, nil); err != nil { - t.Fatal(err) - } - if exists(journalPath(target)) { - t.Fatal("the journal outlived the archive") - } -} diff --git a/internal/apply/archive_removal_test.go b/internal/apply/archive_removal_test.go index b9f7b67..19475dd 100644 --- a/internal/apply/archive_removal_test.go +++ b/internal/apply/archive_removal_test.go @@ -2,6 +2,7 @@ package apply import ( "context" + "encoding/json" "os" "path/filepath" "strings" @@ -152,6 +153,98 @@ func TestAnArchiveDoesNotWriteOverAFileItFound(t *testing.T) { if entries, _ := os.ReadDir(dir); len(entries) != 1 { t.Fatalf("%d entries beside the directory", len(entries)) } + + // Issue 1585: what the host unpacked is written down at once, in its own state directory, so an archive whose + // record in the state was lost is still the host's: the next version replaces its files, drops the ones it no + // longer has and leaves a file somebody added. A file the host never unpacked still refuses it, and so does a + // journal naming a path outside the archive's directory. The journal goes with the archive, record lost or not. + for _, row := range []struct { + name string + journaled bool // the first version was unpacked by the host + foreign string + forged func(j *unpackJournal) + refused string + }{ + {"unpacked by the host, record lost", true, "", nil, ""}, + {"unpacked by the host, somebody's file beside", true, "mine.txt", nil, ""}, + {"never unpacked by the host", false, "", nil, "did not put there"}, + {"a journal naming a path outside", true, "", func(j *unpackJournal) { + j.Unpacked.Files = append(j.Unpacked.Files, "../../etc/passwd") + }, "not inside it"}, + {"a journal naming a parent not above", true, "", func(j *unpackJournal) { + j.Unpacked.Parents = []string{"/etc"} + }, "not above it"}, + } { + t.Run(row.name, func(t *testing.T) { + dir, state := t.TempDir(), t.TempDir() + windows, journals := WindowsIn(state), filepath.Join(state, JournalDirName) + applyIt := func(decl string, known store.State) (store.State, error) { + _, after, err := ApplyMindingWindows(context.Background(), archHost(t), declare(t, decl), known, + store.OriginDeclared, noServices, nil, nil, nil, windows) + return after, err + } + target := filepath.Join(dir, "tools") + first, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "one", "old": "o"}, "") + if row.journaled { + if _, err := applyIt(first, store.State{}); err != nil { + t.Fatal(err) + } + if entries, _ := os.ReadDir(dir); len(entries) != 1 { + t.Fatalf("%d entries beside the archive: the journal is the state directory's", len(entries)) + } + info, err := os.Stat(journalPath(journals, target)) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("no journal of root's alone after the unpack: %v %v", info, err) + } + } else { + os.MkdirAll(target, 0o755) + os.WriteFile(filepath.Join(target, "bin"), []byte("one"), 0o644) + } + if row.forged != nil { + j, _, err := journalOf(journals, target) + if err != nil { + t.Fatal(err) + } + row.forged(&j) + raw, _ := json.Marshal(j) + os.WriteFile(journalPath(journals, target), raw, 0o600) + } + if row.foreign != "" { + os.WriteFile(filepath.Join(target, row.foreign), []byte("x"), 0o644) + } + second, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "two"}, "") + // The state is empty: the record of the first unpack was lost. + after, err := applyIt(second, store.State{}) + if row.refused != "" { + if err == nil || !strings.Contains(err.Error(), row.refused) { + t.Fatalf("refused? %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(target, "bin")); string(got) != "two" { + t.Fatalf("bin is %q", got) + } + if exists(filepath.Join(target, "old")) { + t.Fatal("a file the new version no longer has is still there") + } + if row.foreign != "" && !exists(filepath.Join(target, row.foreign)) { + t.Fatal("somebody's file went") + } + // Taken off, with a record that no longer says what it unpacked: the journal goes too. + for i := range after.Resources { + after.Resources[i].Unpacked = nil + } + if _, err := applyIt(`{"id":"other","type":"file","path":"`+dir+`/other","content":"o"}`, after); err != nil { + t.Fatal(err) + } + if exists(journalPath(journals, target)) { + t.Fatal("the journal outlived the archive") + } + }) + } } func TestADirectoryTheHostMadeGoesOnlyWhenEmpty(t *testing.T) { diff --git a/internal/apply/archive_replace_test.go b/internal/apply/archive_replace_test.go index 8b3431b..2982f1f 100644 --- a/internal/apply/archive_replace_test.go +++ b/internal/apply/archive_replace_test.go @@ -3,7 +3,6 @@ package apply import ( "context" "os" - "strings" "testing" "github.com/novox/mesh-host/internal/store" @@ -33,7 +32,7 @@ func TestAnArchiveReplacesTheTreeItWasUnpackedOver(t *testing.T) { if _, err := os.Stat(dir + "/tools/node_modules"); err == nil { t.Fatal("the previous archive's directory is still there") } - entries := besideTheTree(dir) + entries, _ := os.ReadDir(dir) if len(entries) != 1 { names := []string{} for _, e := range entries { @@ -63,19 +62,7 @@ func TestARefusedArchiveLeavesTheTreeItWouldHaveReplaced(t *testing.T) { if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "old" { t.Fatalf("index.js is %q after a refused archive", got) } - if entries := besideTheTree(dir); len(entries) != 1 { + if entries, _ := os.ReadDir(dir); len(entries) != 1 { t.Fatalf("%d entries beside the tree after a refused archive", len(entries)) } } - -// besideTheTree is what sits in dir, but for the journals of what archives unpacked (novox/hq issue 1585). -func besideTheTree(dir string) []os.DirEntry { - all, _ := os.ReadDir(dir) - var out []os.DirEntry - for _, e := range all { - if !strings.HasSuffix(e.Name(), journalSuffix) { - out = append(out, e) - } - } - return out -} diff --git a/internal/apply/archive_unpacked.go b/internal/apply/archive_unpacked.go index 5038b67..6cbce37 100644 --- a/internal/apply/archive_unpacked.go +++ b/internal/apply/archive_unpacked.go @@ -364,12 +364,13 @@ func mergeInto(fresh, path, owner string, files, dirs []string, o ours) (store.U // module's tools from its bundle — sees the whole tree or none of it, never half, and a file it has // open stays readable until it closes it. The runtime is told the module went by its own membership, // not by the files disappearing. -func removeArchive(a store.Applied) (string, string, error) { - // First, so the parents the host made are empty by the time they are looked at. - if !store.IsFormer(a.ID) && a.Unpacked != nil { - os.Remove(journalPath(filepath.Clean(a.Target))) +func removeArchive(a store.Applied, journals string) (string, string, error) { + action, detail, err := removeArchiveFiles(a) + // And its journal (issue 1585), once the record goes: whether the record said what it unpacked or not. + if err == nil { + dropJournal(journals, a.Target) } - return removeArchiveFiles(a) + return action, detail, err } func removeArchiveFiles(a store.Applied) (string, string, error) { diff --git a/internal/apply/home_links_test.go b/internal/apply/home_links_test.go index 97a47e3..aad8d10 100644 --- a/internal/apply/home_links_test.go +++ b/internal/apply/home_links_test.go @@ -159,7 +159,7 @@ func TestRemovalThroughALinkUnderAHomeIsRefused(t *testing.T) { home, outside := aLinkedHome(t) link(t, outside, filepath.Join(home, ".claude")) _, _, err := remove(context.Background(), archHost(t), store.Applied{ID: "m.f", Type: "file", - Target: filepath.Join(home, ".claude", "shadow")}, nil, nil) + Target: filepath.Join(home, ".claude", "shadow")}, nil, nil, "") mustBeLinkRefusal(t, err) if _, err := os.Stat(filepath.Join(outside, "shadow")); err != nil { t.Fatal("removed through the link") diff --git a/module.json b/module.json index 57270a3..b71573b 100644 --- a/module.json +++ b/module.json @@ -49,7 +49,7 @@ "type": "file", "path": "/usr/lib/nox-mesh-host/launch", "mode": "0755", - "content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A\n# delivered host that is not the known-good one runs on trial: it must report its declaration\n# under its own build — which it marks by writing itself into known-good — within a bound. One\n# that crashes repeatedly, exits without standing aside for anything, or does not report in\n# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host\n# says every record on its reports, so the mesh raises it as a condition. One rollback per\n# version: a version in `rolled-back` is never started again; a newer delivery is.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything — which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares\n# no code with the host and calls none of it: a binary that cannot start cannot be its own\n# recovery.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\n# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.\nBOUND=\"${MESH_HOST_TRIAL_BOUND:-600}\"\nTICK=\"${MESH_HOST_TRIAL_TICK:-5}\"\nGRACE=\"${MESH_HOST_STOP_GRACE:-20}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\nKNOWN_GOOD=\"$STATE_DIR/known-good\"\n# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read\n# by the host (internal/upgrade) and said on its reports.\nROLLED=\"$STATE_DIR/rolled-back\"\nTRIAL=\"$STATE_DIR/trial\"\nEXPIRED=\"$STATE_DIR/trial-expired\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\nnow() { date +%s; }\n\nknown_good() { tr -d '[:space:]' < \"$KNOWN_GOOD\" 2>/dev/null || true; }\n\ndelivered() { [ -n \"$1\" ] && [ -x \"$VERSIONS/$1/$BINARY\" ]; }\n\nrolled_back() { [ -s \"$ROLLED\" ] && cut -f1 \"$ROLLED\" 2>/dev/null | grep -qxF \"$1\"; }\n\n# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.\nversion_of() {\n\tcase \"$1\" in\n\t\t\"$VERSIONS\"/*/\"$BINARY\") v=\"${1#\"$VERSIONS\"/}\"; echo \"${v%/\"$BINARY\"}\" ;;\n\t\t*) echo \"\" ;;\n\tesac\n}\n\nrecord() { # from to outcome why\n\tprintf '%s\\t%s\\t%s\\t%s\\t%s\\n' \"$1\" \"$2\" \"$(now)\" \"$3\" \"$4\" >> \"$ROLLED\"\n}\n\n# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin\n# stands, the version it pinned — or the one placed by hand when nothing has been delivered.\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW — resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort — \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\n#\n# **A pin stands until something newer arrives.** A version delivered after the pin was written, and\n# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.\npick_host() {\n\tnewest=\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tdelivered \"$candidate\" || continue\n\t\trolled_back \"$candidate\" && continue\n\t\tnewest=\"$candidate\"\n\t\tbreak\n\tdone\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$newest\" ] && [ \"$newest\" != \"$pinned\" ] && [ \"$VERSIONS/$newest\" -nt \"$PINNED\" ]; then\n\t\t\tsay \"host $newest arrived after the rollback to $pinned; running it\"\n\t\t\trm -f \"$PINNED\"\n\t\telif delivered \"$pinned\"; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\telse\n\t\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\t\tfi\n\tfi\n\tif [ -n \"$newest\" ]; then\n\t\techo \"$VERSIONS/$newest/$BINARY\"\n\t\treturn 0\n\tfi\n\techo \"$FALLBACK\"\n}\n\n# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.\nroll_back() { # from why\n\tkg=\"$(known_good)\"\n\tif [ -z \"$1\" ] || [ \"$1\" = \"$kg\" ] || ! delivered \"$kg\"; then\n\t\treturn 1\n\tfi\n\trecord \"$1\" \"$kg\" rolled-back \"$2\"\n\t# The pin is what stops the launcher starting a version newer than known-good that is not the one\n\t# rolled back; the record is what stops it starting this one again.\n\tprintf '%s\\n' \"$kg\" > \"$PINNED\"\n\trm -f \"$TRIAL\"\n\tsay \"rolled back from host $1 to $kg: $2\"\n\treturn 0\n}\n\n# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.\n# A subshell beside the host, so the launcher's own wait is untouched.\ntrial_watch() { # pid version deadline\n\twhile kill -0 \"$1\" 2>/dev/null; do\n\t\t[ \"$(known_good)\" = \"$2\" ] && return 0\n\t\tif [ \"$(now)\" -ge \"$3\" ]; then\n\t\t\tprintf '%s\\n' \"$2\" > \"$EXPIRED\"\n\t\t\tsay \"host $2 did not report within ${BOUND}s of starting; stopping it\"\n\t\t\tkill -TERM \"$1\" 2>/dev/null\n\t\t\twaited=0\n\t\t\twhile kill -0 \"$1\" 2>/dev/null && [ \"$waited\" -lt \"$GRACE\" ]; do\n\t\t\t\tsleep 1\n\t\t\t\twaited=$((waited + 1))\n\t\t\tdone\n\t\t\tkill -KILL \"$1\" 2>/dev/null\n\t\t\treturn 0\n\t\tfi\n\t\tsleep \"$TICK\"\n\tdone\n}\n\nchild=\nwatcher=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at\n# the next boot.\nSELF=\"$(cksum < \"$0\" 2>/dev/null || true)\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is\n\t# the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tHOST=\"$(pick_host)\"\n\tversion=\"$(version_of \"$HOST\")\"\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif roll_back \"$version\" \"it failed $count times in a row\"; then\n\t\t\t# Fresh count for the version now run: it deserves its own attempts, and without this\n\t\t\t# it inherits a count already over the limit and halts at once. The variable too, not\n\t\t\t# only the file: resetting one and not the other made the next failure count from the\n\t\t\t# OLD value — so the rolled-back version got one attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\t\tcontinue\n\t\tfi\n\t\tkg=\"$(known_good)\"\n\t\tif [ -n \"$kg\" ] && { [ \"$version\" = \"$kg\" ] || [ -z \"$version\" ]; } && [ -s \"$ROLLED\" ]; then\n\t\t\t# Already gone back, and what it went back to fails as well: this is the machine and\n\t\t\t# not a binary. Rolling back again would flap between two versions for ever.\n\t\t\tsay \"the host failed $count times after a rollback. the version it went back to does\"\n\t\t\tsay \"not start either, so this is the machine and not the binary.\"\n\t\t\trecord \"${version:-placed-by-hand}\" \"\" halted \"the version rolled back to failed $count times as well\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\t\t# Nothing to go back to: no host has ever reported here, or the one that did was placed by\n\t\t# hand and is not delivered. An installation failure rather than an upgrade's — said, and\n\t\t# tried again slowly, never guessed at.\n\t\tsay \"the host failed $count times and there is no delivered known-good version to go back to.\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\tfi\n\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\n\t# **On trial**: a delivered version that is not the known-good one, while a known-good one is\n\t# delivered to go back to. The trial starts when this version was first started, and survives\n\t# this launcher restarting.\n\ttrial=\n\tdeadline=\n\tkg=\"$(known_good)\"\n\tif [ -n \"$version\" ] && [ \"$version\" != \"$kg\" ] && delivered \"$kg\"; then\n\t\ttrial=yes\n\t\tstarted=\n\t\tif [ -s \"$TRIAL\" ]; then\n\t\t\tread -r on since _ < \"$TRIAL\" 2>/dev/null || on=\n\t\t\t[ \"${on:-}\" = \"$version\" ] && started=\"${since:-}\"\n\t\tfi\n\t\tcase \"$started\" in\n\t\t\t'' | *[!0-9]*) started=\"$(now)\"; printf '%s %s\\n' \"$version\" \"$started\" > \"$TRIAL\" ;;\n\t\tesac\n\t\tdeadline=$((started + BOUND))\n\t\tif [ \"$(now)\" -ge \"$deadline\" ]; then\n\t\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\" && continue\n\t\tfi\n\telse\n\t\trm -f \"$TRIAL\"\n\tfi\n\n\trm -f \"$EXPIRED\"\n\tsay \"running $HOST${trial:+ (on trial until it reports)}\"\n\t\"$HOST\" run &\n\tchild=$!\n\twatcher=\n\tif [ -n \"$trial\" ]; then\n\t\ttrial_watch \"$child\" \"$version\" \"$deadline\" &\n\t\twatcher=$!\n\tfi\n\tstatus=0\n\twait \"$child\" || status=$?\n\tif [ -n \"$stopping\" ]; then\n\t\t# The signal interrupted the wait, not the host: it was told, and is finishing what it was\n\t\t# doing. Waited for, so the service manager does not kill it half way through an apply.\n\t\twait \"$child\" 2>/dev/null\n\tfi\n\tchild=\n\tif [ -n \"$watcher\" ]; then\n\t\tkill \"$watcher\" 2>/dev/null\n\t\twait \"$watcher\" 2>/dev/null\n\t\twatcher=\n\tfi\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -n \"$trial\" ] && [ -s \"$EXPIRED\" ] && [ \"$(cat \"$EXPIRED\" 2>/dev/null)\" = \"$version\" ]; then\n\t\trm -f \"$EXPIRED\"\n\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\tcontinue\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\t#\n\t\t\t# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the\n\t\t\t# same host is still the one to run has not stood aside for a successor: it has\n\t\t\t# stopped, and a host that stops at once would otherwise loop here for ever unseen.\n\t\t\tif [ -n \"$trial\" ] && [ \"$(pick_host)\" = \"$HOST\" ] && [ \"$(known_good)\" != \"$version\" ]; then\n\t\t\t\tsay \"host $version exited cleanly on trial with nothing newer to stand aside for\"\n\t\t\telse\n\t\t\t\t# A delivered successor of this launcher runs from here on, not from the next boot.\n\t\t\t\tif [ -n \"$SELF\" ] && [ \"$(cksum < \"$0\" 2>/dev/null || true)\" != \"$SELF\" ]; then\n\t\t\t\t\tsay \"the launcher was replaced; running the new one\"\n\t\t\t\t\texec \"$0\"\n\t\t\t\tfi\n\t\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\t\tcontinue\n\t\t\tfi\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n" + "content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A\n# delivered host that is not the known-good one runs on trial: it must report its declaration\n# under its own build — which it marks by writing itself into known-good — within a bound. One\n# that crashes repeatedly, exits without standing aside for anything, or does not report in\n# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host\n# says every record on its reports, so the mesh raises it as a condition. One rollback per\n# version: a version in `rolled-back` is never started again; a newer delivery is.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything — which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares\n# no code with the host and calls none of it: a binary that cannot start cannot be its own\n# recovery.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\n# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.\nBOUND=\"${MESH_HOST_TRIAL_BOUND:-600}\"\nTICK=\"${MESH_HOST_TRIAL_TICK:-5}\"\n# How long a host told to stop is waited for before it is killed, in seconds: past the 60 s the host itself waits for\n# the apply in flight to end and save (novox/hq issue 1585), so a stop never cuts a save off.\nGRACE=\"${MESH_HOST_STOP_GRACE:-75}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\nKNOWN_GOOD=\"$STATE_DIR/known-good\"\n# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read\n# by the host (internal/upgrade) and said on its reports.\nROLLED=\"$STATE_DIR/rolled-back\"\nTRIAL=\"$STATE_DIR/trial\"\nEXPIRED=\"$STATE_DIR/trial-expired\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\nnow() { date +%s; }\n\nknown_good() { tr -d '[:space:]' < \"$KNOWN_GOOD\" 2>/dev/null || true; }\n\ndelivered() { [ -n \"$1\" ] && [ -x \"$VERSIONS/$1/$BINARY\" ]; }\n\nrolled_back() { [ -s \"$ROLLED\" ] && cut -f1 \"$ROLLED\" 2>/dev/null | grep -qxF \"$1\"; }\n\n# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.\nversion_of() {\n\tcase \"$1\" in\n\t\t\"$VERSIONS\"/*/\"$BINARY\") v=\"${1#\"$VERSIONS\"/}\"; echo \"${v%/\"$BINARY\"}\" ;;\n\t\t*) echo \"\" ;;\n\tesac\n}\n\nrecord() { # from to outcome why\n\tprintf '%s\\t%s\\t%s\\t%s\\t%s\\n' \"$1\" \"$2\" \"$(now)\" \"$3\" \"$4\" >> \"$ROLLED\"\n}\n\n# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin\n# stands, the version it pinned — or the one placed by hand when nothing has been delivered.\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW — resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort — \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\n#\n# **A pin stands until something newer arrives.** A version delivered after the pin was written, and\n# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.\npick_host() {\n\tnewest=\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tdelivered \"$candidate\" || continue\n\t\trolled_back \"$candidate\" && continue\n\t\tnewest=\"$candidate\"\n\t\tbreak\n\tdone\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$newest\" ] && [ \"$newest\" != \"$pinned\" ] && [ \"$VERSIONS/$newest\" -nt \"$PINNED\" ]; then\n\t\t\tsay \"host $newest arrived after the rollback to $pinned; running it\"\n\t\t\trm -f \"$PINNED\"\n\t\telif delivered \"$pinned\"; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\telse\n\t\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\t\tfi\n\tfi\n\tif [ -n \"$newest\" ]; then\n\t\techo \"$VERSIONS/$newest/$BINARY\"\n\t\treturn 0\n\tfi\n\techo \"$FALLBACK\"\n}\n\n# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.\nroll_back() { # from why\n\tkg=\"$(known_good)\"\n\tif [ -z \"$1\" ] || [ \"$1\" = \"$kg\" ] || ! delivered \"$kg\"; then\n\t\treturn 1\n\tfi\n\trecord \"$1\" \"$kg\" rolled-back \"$2\"\n\t# The pin is what stops the launcher starting a version newer than known-good that is not the one\n\t# rolled back; the record is what stops it starting this one again.\n\tprintf '%s\\n' \"$kg\" > \"$PINNED\"\n\trm -f \"$TRIAL\"\n\tsay \"rolled back from host $1 to $kg: $2\"\n\treturn 0\n}\n\n# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.\n# A subshell beside the host, so the launcher's own wait is untouched.\ntrial_watch() { # pid version deadline\n\twhile kill -0 \"$1\" 2>/dev/null; do\n\t\t[ \"$(known_good)\" = \"$2\" ] && return 0\n\t\tif [ \"$(now)\" -ge \"$3\" ]; then\n\t\t\tprintf '%s\\n' \"$2\" > \"$EXPIRED\"\n\t\t\tsay \"host $2 did not report within ${BOUND}s of starting; stopping it\"\n\t\t\tkill -TERM \"$1\" 2>/dev/null\n\t\t\twaited=0\n\t\t\twhile kill -0 \"$1\" 2>/dev/null && [ \"$waited\" -lt \"$GRACE\" ]; do\n\t\t\t\tsleep 1\n\t\t\t\twaited=$((waited + 1))\n\t\t\tdone\n\t\t\tkill -KILL \"$1\" 2>/dev/null\n\t\t\treturn 0\n\t\tfi\n\t\tsleep \"$TICK\"\n\tdone\n}\n\nchild=\nwatcher=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at\n# the next boot.\nSELF=\"$(cksum < \"$0\" 2>/dev/null || true)\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is\n\t# the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tHOST=\"$(pick_host)\"\n\tversion=\"$(version_of \"$HOST\")\"\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif roll_back \"$version\" \"it failed $count times in a row\"; then\n\t\t\t# Fresh count for the version now run: it deserves its own attempts, and without this\n\t\t\t# it inherits a count already over the limit and halts at once. The variable too, not\n\t\t\t# only the file: resetting one and not the other made the next failure count from the\n\t\t\t# OLD value — so the rolled-back version got one attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\t\tcontinue\n\t\tfi\n\t\tkg=\"$(known_good)\"\n\t\tif [ -n \"$kg\" ] && { [ \"$version\" = \"$kg\" ] || [ -z \"$version\" ]; } && [ -s \"$ROLLED\" ]; then\n\t\t\t# Already gone back, and what it went back to fails as well: this is the machine and\n\t\t\t# not a binary. Rolling back again would flap between two versions for ever.\n\t\t\tsay \"the host failed $count times after a rollback. the version it went back to does\"\n\t\t\tsay \"not start either, so this is the machine and not the binary.\"\n\t\t\trecord \"${version:-placed-by-hand}\" \"\" halted \"the version rolled back to failed $count times as well\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\t\t# Nothing to go back to: no host has ever reported here, or the one that did was placed by\n\t\t# hand and is not delivered. An installation failure rather than an upgrade's — said, and\n\t\t# tried again slowly, never guessed at.\n\t\tsay \"the host failed $count times and there is no delivered known-good version to go back to.\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\tfi\n\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\n\t# **On trial**: a delivered version that is not the known-good one, while a known-good one is\n\t# delivered to go back to. The trial starts when this version was first started, and survives\n\t# this launcher restarting.\n\ttrial=\n\tdeadline=\n\tkg=\"$(known_good)\"\n\tif [ -n \"$version\" ] && [ \"$version\" != \"$kg\" ] && delivered \"$kg\"; then\n\t\ttrial=yes\n\t\tstarted=\n\t\tif [ -s \"$TRIAL\" ]; then\n\t\t\tread -r on since _ < \"$TRIAL\" 2>/dev/null || on=\n\t\t\t[ \"${on:-}\" = \"$version\" ] && started=\"${since:-}\"\n\t\tfi\n\t\tcase \"$started\" in\n\t\t\t'' | *[!0-9]*) started=\"$(now)\"; printf '%s %s\\n' \"$version\" \"$started\" > \"$TRIAL\" ;;\n\t\tesac\n\t\tdeadline=$((started + BOUND))\n\t\tif [ \"$(now)\" -ge \"$deadline\" ]; then\n\t\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\" && continue\n\t\tfi\n\telse\n\t\trm -f \"$TRIAL\"\n\tfi\n\n\trm -f \"$EXPIRED\"\n\tsay \"running $HOST${trial:+ (on trial until it reports)}\"\n\t\"$HOST\" run &\n\tchild=$!\n\twatcher=\n\tif [ -n \"$trial\" ]; then\n\t\ttrial_watch \"$child\" \"$version\" \"$deadline\" &\n\t\twatcher=$!\n\tfi\n\tstatus=0\n\twait \"$child\" || status=$?\n\tif [ -n \"$stopping\" ]; then\n\t\t# The signal interrupted the wait, not the host: it was told, and is finishing what it was\n\t\t# doing. Waited for, so the service manager does not kill it half way through an apply.\n\t\twait \"$child\" 2>/dev/null\n\tfi\n\tchild=\n\tif [ -n \"$watcher\" ]; then\n\t\tkill \"$watcher\" 2>/dev/null\n\t\twait \"$watcher\" 2>/dev/null\n\t\twatcher=\n\tfi\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -n \"$trial\" ] && [ -s \"$EXPIRED\" ] && [ \"$(cat \"$EXPIRED\" 2>/dev/null)\" = \"$version\" ]; then\n\t\trm -f \"$EXPIRED\"\n\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\tcontinue\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\t#\n\t\t\t# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the\n\t\t\t# same host is still the one to run has not stood aside for a successor: it has\n\t\t\t# stopped, and a host that stops at once would otherwise loop here for ever unseen.\n\t\t\tif [ -n \"$trial\" ] && [ \"$(pick_host)\" = \"$HOST\" ] && [ \"$(known_good)\" != \"$version\" ]; then\n\t\t\t\tsay \"host $version exited cleanly on trial with nothing newer to stand aside for\"\n\t\t\telse\n\t\t\t\t# A delivered successor of this launcher runs from here on, not from the next boot.\n\t\t\t\tif [ -n \"$SELF\" ] && [ \"$(cksum < \"$0\" 2>/dev/null || true)\" != \"$SELF\" ]; then\n\t\t\t\t\tsay \"the launcher was replaced; running the new one\"\n\t\t\t\t\texec \"$0\"\n\t\t\t\tfi\n\t\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\t\tcontinue\n\t\t\tfi\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n" }, { "id": "next", diff --git a/packaging/nox-mesh-host-launch b/packaging/nox-mesh-host-launch index e75d59a..cf946a2 100755 --- a/packaging/nox-mesh-host-launch +++ b/packaging/nox-mesh-host-launch @@ -38,7 +38,9 @@ BACKOFF="${MESH_HOST_BACKOFF:-5}" # How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply. BOUND="${MESH_HOST_TRIAL_BOUND:-600}" TICK="${MESH_HOST_TRIAL_TICK:-5}" -GRACE="${MESH_HOST_STOP_GRACE:-20}" +# How long a host told to stop is waited for before it is killed, in seconds: past the 60 s the host itself waits for +# the apply in flight to end and save (novox/hq issue 1585), so a stop never cuts a save off. +GRACE="${MESH_HOST_STOP_GRACE:-75}" ONCE="${MESH_HOST_RUN_ONCE:-}" # tests run one iteration; nothing else sets this ATTEMPTS="$STATE_DIR/start-attempts" diff --git a/packaging/nox-mesh-host.openrc b/packaging/nox-mesh-host.openrc index cc740f0..7e96b88 100644 --- a/packaging/nox-mesh-host.openrc +++ b/packaging/nox-mesh-host.openrc @@ -5,4 +5,6 @@ name="nox-mesh-host" command="/usr/lib/nox-mesh-host/launch" supervisor="supervise-daemon" +# The same stop bound as the systemd unit's TimeoutStopSec (issue 1585): TERM, 120 s, then KILL. +retry="TERM/120/KILL/5" depend() { need net; } diff --git a/packaging/nox-mesh-host.service b/packaging/nox-mesh-host.service index c9c6df1..05e97dc 100644 --- a/packaging/nox-mesh-host.service +++ b/packaging/nox-mesh-host.service @@ -12,6 +12,9 @@ Restart=always RestartSec=5s StateDirectory=mesh-host KillMode=mixed +# Past the launcher's own stop grace (75 s), which is past the 60 s the host waits for the apply in +# flight to end and save (novox/hq issue 1585): the service manager's kill is the last resort. +TimeoutStopSec=120 [Install] WantedBy=multi-user.target