Roll a core build back by a witness that is not the new build (hq to-be 45 Phase 4)

The launcher trusted a counter only a by-hand reconcile ever cleared and a
known-good nothing in the daemon wrote, so no machine could roll its host back;
the controller and the node tools were replaced in place with nothing kept.

- The launcher runs a delivered host that is not known-good on trial: one that
  crashes, stops for nothing, or does not report within ten minutes goes back
  to known-good, once per version, recorded in rolled-back. The host proves
  itself when the mesh takes a report under its own build, says every standing
  verdict on its reports, never stands aside for a rolled-back version, and
  restarts its service once when its launcher was replaced on disk.
- The engine keeps the controller's and the node tools' previous build beside
  the new one and judges the new one: the lease taken by the controller it
  started (read-only direct get of mesh-controller_lease/holder), or this
  machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds
  of time it could ask. Not healthy: the previous restored, once, said. Proved:
  the previous deleted. A build declared not-reversible is never rolled back.
- Retire never removes a version newer than the running one.

internal/apply/process.go

@@ -8,9 +8,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/witness"
 )
 
 // Running the mesh's own code, without the module choosing how.
@@ -105,20 +107,50 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		return out, nil
 	}
 
-	// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
-	// previous version would be loaded by a runtime that walks a directory.
-	if err := os.RemoveAll(at); err != nil {
-		return out, err
+	// **A core process keeps the build before it until the new one is proved** (novox/hq to-be 45
+	// §8): the engine's witness judges the new build and restores the kept one when it is not healthy
+	// in bound. Placing decides what happens to the directory first — the running build moved aside,
+	// a build on trial discarded, a build rolled back here refused, the running build kept when it is
+	// the one declared — and keeps that, so a restoration later knows which build is where.
+	by := witnessOf(r)
+	placing := witness.Placing{Unpack: true}
+	if by == witness.ByNone {
+		// Nothing judged. Whatever was kept for a process that was judged before goes.
+		if err := witness.Forget(daemonRoot, r.Name); err != nil {
+			return out, err
+		}
+		// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
+		// previous version would be loaded by a runtime that walks a directory.
+		if err := os.RemoveAll(at); err != nil {
+			return out, err
+		}
+	} else {
+		placing, err = witness.Place(daemonRoot, r.Name, by, got, digestWritten(previous.Wrote), r.NotReversible, time.Now())
+		if err != nil {
+			return out, err
+		}
 	}
-	if err := os.MkdirAll(at, 0o755); err != nil {
-		return out, err
+	written := 0
+	if placing.Unpack {
+		if err := os.MkdirAll(at, 0o755); err != nil {
+			return out, err
+		}
+		if written, err = unpack(body, at); err != nil {
+			return out, err
+		}
+		if err := ownAll(at, r.User); err != nil {
+			return out, err
+		}
 	}
-	written, err := unpack(body, at)
-	if err != nil {
-		return out, err
-	}
-	if err := ownAll(at, r.User); err != nil {
-		return out, err
+	if placing.Commit != nil {
+		// Kept whatever the start below does: a build that would not start is still the build placed,
+		// and it is the witness's to judge.
+		defer func() {
+			if err := placing.Commit(); err != nil {
+				fmt.Fprintf(os.Stderr, "mesh-host: placed %s, and what the witness keeps about it could not be saved: %v\n",
+					r.Name, err)
+			}
+		}()
 	}
 
 	// **A step is run to completion, not installed.** What follows it is gated on it finishing,
@@ -201,9 +233,15 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		out.Action = "created"
 	}
 	out.Detail = fmt.Sprintf("%d file(s), running as %s.service", written, r.Name)
+	if !placing.Unpack {
+		out.Detail = fmt.Sprintf("its build already in place, running as %s.service", r.Name)
+	}
 	if because != "" {
 		out.Detail += ", restarted because " + because + " changed"
 	}
+	if placing.Detail != "" {
+		out.Detail += "; " + placing.Detail
+	}
 	out.wrote = want
 	return out, nil
 }
@@ -358,6 +396,13 @@ func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, st
 		}
 	}
 	bundle := filepath.Join(daemonRoot, name)
+	// And the build kept before it, with what the witness knew (novox/hq to-be 45 §8).
+	if _, err := os.Stat(witness.Dir(daemonRoot, name)); err == nil {
+		found = true
+		if err := witness.Forget(daemonRoot, name); err != nil {
+			return "", "", err
+		}
+	}
 	if _, err := os.Stat(bundle); err == nil {
 		found = true
 		if err := os.RemoveAll(bundle); err != nil {
@@ -381,3 +426,27 @@ func runFrom(r *declaration.Process) []string {
 	}
 	return run
 }
+
+// witnessOf is how a process's new builds are judged: as it declares, or by its name's default — the
+// mesh's two core processes (novox/hq to-be 45 §8). A step or a scheduled run is never judged.
+func witnessOf(r *declaration.Process) string {
+	if r.RunOnce || r.Schedule != "" {
+		return witness.ByNone
+	}
+	if r.Witness != "" {
+		return r.Witness
+	}
+	return witness.Default(r.Name)
+}
+
+// digestWritten is the bundle digest a process's record says was placed: the first half of what it
+// wrote, "sha256:<hex> <unit>".
+func digestWritten(wrote string) string {
+	if fields := strings.Fields(wrote); len(fields) > 0 && strings.HasPrefix(fields[0], "sha256:") {
+		return fields[0]
+	}
+	return ""
+}
+
+// DaemonRoot is where unpacked daemons live — and what a witness keeps beside them.
+func DaemonRoot() string { return daemonRoot }