Roll a core build back by a witness that is not the new build (hq to-be 45 Phase 4)

The launcher trusted a counter only a by-hand reconcile ever cleared and a
known-good nothing in the daemon wrote, so no machine could roll its host back;
the controller and the node tools were replaced in place with nothing kept.

- The launcher runs a delivered host that is not known-good on trial: one that
  crashes, stops for nothing, or does not report within ten minutes goes back
  to known-good, once per version, recorded in rolled-back. The host proves
  itself when the mesh takes a report under its own build, says every standing
  verdict on its reports, never stands aside for a rolled-back version, and
  restarts its service once when its launcher was replaced on disk.
- The engine keeps the controller's and the node tools' previous build beside
  the new one and judges the new one: the lease taken by the controller it
  started (read-only direct get of mesh-controller_lease/holder), or this
  machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds
  of time it could ask. Not healthy: the previous restored, once, said. Proved:
  the previous deleted. A build declared not-reversible is never rolled back.
- Retire never removes a version newer than the running one.

cmd/mesh-host/main.go

@@ -648,8 +648,10 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	//
 	// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
 	// hiding it would make a machine quietly fill up.
-	if version != "" {
-		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil {
+	// Asked with the version this host RUNS, read from where it sits: the link-time stamp names no
+	// delivered version, and retiring around a name that is not there would remove the one running.
+	if v := runningVersion(); v != "" {
+		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), v); err != nil {
 			fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
 		} else if len(retired) > 0 {
 			fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
@@ -1078,7 +1080,7 @@ func runLink(ctx context.Context, opts options) error {
 		// a delivered host never matched the newest delivered version, so it stood aside on every
 		// push for ever, and standing aside then cancelled the report, so the mesh never heard from it
 		// again (novox/hq 04-ISSUES/163).
-		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion()); {
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion(), rolledBackHosts(opts.state)...); {
 		case err != nil:
 			// Said, not fatal. A host that cannot read the delivered versions is still running this
 			// machine correctly; what it has lost is the ability to be replaced.
@@ -1102,6 +1104,7 @@ func runLink(ctx context.Context, opts options) error {
 	// its firewall, its outward links — are said by the same worker, in the order they are made
 	// (novox/hq ADR 0100, issue 267).
 	watch := &adoptionWatch{}
+	proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
 	queue := &link.Queue{
 		Membership: membership,
 		Apply:      applier,
@@ -1109,7 +1112,11 @@ func runLink(ctx context.Context, opts options) error {
 		News:       func(r link.Report) bool { return worthSaying(r) && watch.differs(r) },
 		// Counted as said only once the broker has taken it: queued and lost — the link down, the
 		// publish refused — the change would never be said again (novox/hq ADR 0100).
-		Heard:   watch.said,
+		// And the first account under this build is what proves it to the launcher (to-be 45 §8).
+		Heard: func(r link.Report) {
+			watch.said(r)
+			proof.heard(r)
+		},
 		Unsaid:  unsaidBeside(opts.state),
 		Numbers: numbersBeside(opts.state),
 		Say:     say,
@@ -1123,8 +1130,18 @@ func runLink(ctx context.Context, opts options) error {
 	// **A host starting is a reason to reconcile** (to-be 45 §6: the self-update hand-over is one of
 	// the four): its scheduled steps are armed from the declaration the node kept by the first apply,
 	// and a successor that waited five minutes for it left them unarmed for five.
-	queue.ReconcileDue()
+	//
+	// **A host on trial says its account whether or not it is news** (to-be 45 §8): the launcher
+	// waits for this build to report its declaration, and a converged machine with nothing new to
+	// say would otherwise not say anything until the mesh next sent it something.
+	if unproved(opts.state, runningVersion()) {
+		queue.ReportAsked()
+	} else {
+		queue.ReconcileDue()
+	}
 	go holdTheMachine(ctx, queue)
+	// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
+	go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
 
 	held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
 	// The act in hand finishes and is kept before this process exits: an apply that stood aside with
@@ -1225,6 +1242,10 @@ func adoptionFingerprint(r link.Report) string {
 	for _, f := range r.Filters {
 		parts = append(parts, "filter "+f.Owner+" "+f.Where+" "+f.Refuses)
 	}
+	// And a witness's verdict (to-be 45 §8): one reached or one ended is said at the next reconcile.
+	for _, v := range r.Rollbacks {
+		parts = append(parts, fmt.Sprintf("rollback %s %s %s %s", v.Component, v.From, v.To, v.Outcome))
+	}
 	if r.FoundFirewall != nil {
 		parts = append(parts, fmt.Sprintf("found-firewall %s active=%v retired-by=%s", r.FoundFirewall.Kind,
 			r.FoundFirewall.Active, r.FoundFirewall.RetiredBy))
@@ -1369,7 +1390,8 @@ func worthSaying(report link.Report) bool {
 		return false
 	}
 	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0 ||
-		len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0
+		len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0 ||
+		len(report.Rollbacks) > 0
 }
 
 // announceOr is what the apply writes its detail with, given what the caller has to say things with.
@@ -1528,6 +1550,7 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 	}
 
 	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(),
+		Rollbacks: standingRollbacks(opts.state), Witness: link.WitnessContract,
 		Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,

cmd/mesh-host/witnessing.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/upgrade"
+	"github.com/novox/mesh-host/internal/witness"
+)
+
+// The node-engine's part in core upgrades that roll back (novox/hq to-be 45 §8, ADR 0227 rule 8):
+// proving its own build to the launcher that witnesses it, witnessing the controller and the node
+// tools it places, and saying every verdict on its reports.
+
+// standingRollbacks is every verdict that still stands on this machine: the launcher's about this
+// host, and the engine's about the processes it witnesses. Said on every report.
+func standingRollbacks(statePath string) []link.Rollback {
+	var out []link.Rollback
+	verdicts, err := upgrade.ReadRolledBack(upgrade.RolledBackPath(statePath))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-host: %v\n", err)
+	}
+	for _, v := range verdicts {
+		out = append(out, link.Rollback{Component: link.ComponentEngine, From: v.From, To: v.To,
+			Outcome: v.Outcome, Why: v.Why, At: v.At})
+	}
+	return append(out, witness.Standing(apply.DaemonRoot())...)
+}
+
+// rolledBackHosts are the host versions the launcher will not start again, so this host never stands
+// aside for one.
+func rolledBackHosts(statePath string) []string {
+	verdicts, _ := upgrade.ReadRolledBack(upgrade.RolledBackPath(statePath))
+	return upgrade.RolledBackVersions(verdicts)
+}
+
+// proving is this host waiting for the mesh to take a report it made under its own build — the
+// evidence its launcher's trial waits for — and acting on it once.
+type proving struct {
+	statePath string
+	version   string
+	say       func(string)
+	once      sync.Once
+}
+
+// unproved says whether this host's version is not yet known-good here: a host on trial.
+func unproved(statePath, version string) bool {
+	known, err := upgrade.ReadKnownGood(upgrade.KnownGoodPath(statePath))
+	return err != nil || known != version
+}
+
+// heard is told every account of the machine the mesh has taken. The first one about a declaration,
+// made by this build, proves it.
+func (p *proving) heard(r link.Report) {
+	if r.Declared == "" || r.Refused != "" || r.Host != p.version {
+		return
+	}
+	p.once.Do(func() {
+		retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version)
+		if err != nil {
+			p.say(fmt.Sprintf("this host reported under its own build %s, and proving it was not complete: %v", p.version, err))
+		}
+		if len(retired) > 0 {
+			p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", ")))
+		}
+		renewLauncher(p.say)
+	})
+}
+
+// The launcher, as the service manager runs it.
+const (
+	launcherPath = upgrade.DefaultLibexec + "/launch"
+	hostUnit     = "nox-mesh-host.service"
+)
+
+// renewLauncher asks the service manager to restart this host's service when the launcher running
+// it was replaced on disk since it started (novox/hq to-be 45 §8): a delivered launcher otherwise
+// takes effect only at the next boot, and the witness it carries with it. A launcher from
+// this one on runs its successor itself, at the host's next clean exit; this is for the launcher
+// before it. Said, and only when it is certain: the parent is the launcher, and the file it reads
+// is gone from under it.
+func renewLauncher(say func(string)) {
+	if _, err := os.Stat("/run/systemd/system"); err != nil {
+		return
+	}
+	if !launcherReplaced(os.Getppid(), "/proc", launcherPath) {
+		return
+	}
+	say("the launcher running this host was replaced on disk; asking the service manager to run the new one")
+	if _, err := apply.ExecRunner(context.Background(), "systemctl", "restart", "--no-block", hostUnit); err != nil {
+		say("could not ask for the new launcher, so it runs from the next boot: " + err.Error())
+	}
+}
+
+// launcherReplaced is whether process pid is a shell reading the launcher at path, from a file that
+// is no longer there — the one a delivery renamed a new launcher over.
+func launcherReplaced(pid int, proc, path string) bool {
+	if pid <= 1 {
+		return false
+	}
+	base := filepath.Join(proc, strconv.Itoa(pid))
+	cmdline, err := os.ReadFile(filepath.Join(base, "cmdline"))
+	if err != nil || !strings.Contains(string(cmdline), path) {
+		return false
+	}
+	fds, err := os.ReadDir(filepath.Join(base, "fd"))
+	if err != nil {
+		return false
+	}
+	for _, fd := range fds {
+		target, err := os.Readlink(filepath.Join(base, "fd", fd.Name()))
+		if err == nil && target == path+" (deleted)" {
+			return true
+		}
+	}
+	return false
+}
+
+// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
+func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue, say func(string)) {
+	host, _ := os.Hostname()
+	w := &witness.Watcher{
+		Root: apply.DaemonRoot(), Node: node, Host: host,
+		Asker: func() link.Asker { return queue.Asker() },
+		Run:   witness.Runner(apply.ExecRunner),
+		Hold: func(f func()) {
+			applying.Lock()
+			defer applying.Unlock()
+			f()
+		},
+		// Said now, not at the next report: the verdict is on every report from here, and the mesh
+		// is asked for one at once.
+		Concluded: func(link.Rollback) { queue.ReportAsked() },
+		Say:       say,
+	}
+	w.Watch(ctx)
+}

cmd/mesh-host/witnessing_test.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/upgrade"
+)
+
+// The first account the mesh takes from this build, about a declaration, proves it to the launcher;
+// nothing else does — a refusal, a report about no declaration, another build's report (to-be 45 §8).
+func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) {
+	state := filepath.Join(t.TempDir(), "state.json")
+	t.Setenv("MESH_HOST_LIBEXEC", t.TempDir())
+	var said []string
+	p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }}
+
+	for _, r := range []link.Report{
+		{Host: "2.0", Refused: "no"},
+		{Host: "2.0"},
+		{Host: "1.0", Declared: "abc"},
+	} {
+		p.heard(r)
+		if !unproved(state, "2.0") {
+			t.Fatalf("%+v proved this build", r)
+		}
+	}
+	p.heard(link.Report{Host: "2.0", Declared: "abc"})
+	if unproved(state, "2.0") {
+		t.Fatalf("an account of a declaration under this build did not prove it (%v)", said)
+	}
+	if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" {
+		t.Fatalf("known-good is %q", got)
+	}
+}
+
+// Every verdict that stands is said, and a reconcile that has one says it unasked.
+func TestAStandingRollbackIsSaidOnEveryReport(t *testing.T) {
+	state := filepath.Join(t.TempDir(), "state.json")
+	line := "2.0\t1.0\t1759744800\trolled-back\tit did not report within 600s of starting\n"
+	if err := os.WriteFile(upgrade.RolledBackPath(state), []byte(line), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	got := standingRollbacks(state)
+	if len(got) == 0 || got[0].Component != link.ComponentEngine || got[0].From != "2.0" || got[0].To != "1.0" ||
+		got[0].Outcome != link.RolledBack {
+		t.Fatalf("what the report says is %+v", got)
+	}
+	if !worthSaying(link.Report{Rollbacks: got}) {
+		t.Fatal("a reconcile with a rollback standing does not say it")
+	}
+	if adoptionFingerprint(link.Report{Rollbacks: got}) == adoptionFingerprint(link.Report{}) {
+		t.Fatal("a rollback reached or ended is not news to the reconcile")
+	}
+}
+
+// The launcher running this host is known to have been replaced only when it certainly was: a shell
+// reading the launcher, from a file renamed over. Asked of a real process, not a model of one.
+func TestAReplacedLauncherIsSeen(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "launch")
+	if err := os.WriteFile(path, []byte("#!/bin/sh\nsleep 5\necho done\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	shell := exec.Command("sh", path)
+	if err := shell.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = shell.Process.Kill(); _ = shell.Wait() })
+	time.Sleep(200 * time.Millisecond)
+	if launcherReplaced(shell.Process.Pid, "/proc", path) {
+		t.Fatal("a launcher still on disk reads as replaced")
+	}
+	// Delivered as the mesh writes a file: a new one renamed over it.
+	if err := os.WriteFile(path+".new", []byte("#!/bin/sh\necho new\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Rename(path+".new", path); err != nil {
+		t.Fatal(err)
+	}
+	if !launcherReplaced(shell.Process.Pid, "/proc", path) {
+		fds, _ := os.ReadDir(filepath.Join("/proc", strconv.Itoa(shell.Process.Pid), "fd"))
+		t.Skipf("this shell does not keep its script open (%d fds), so a replaced launcher cannot be seen here", len(fds))
+	}
+	if launcherReplaced(shell.Process.Pid, "/proc", filepath.Join(dir, "other")) {
+		t.Fatal("a process reading another script reads as this launcher")
+	}
+}