Roll a core build back by a witness that is not the new build (hq to-be 45 Phase 4)

The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
2026-10-06 18:23:56 +02:00
parent 8d853791b2
commit 0c405b70cc
23 changed files with 2636 additions and 115 deletions
@@ -8,9 +8,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"

 	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/witness"
 )

 // Running the mesh's own code, without the module choosing how.
@@ -105,20 +107,50 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		return out, nil
 	}

-	// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
-	// previous version would be loaded by a runtime that walks a directory.
-	if err := os.RemoveAll(at); err != nil {
-		return out, err
+	// **A core process keeps the build before it until the new one is proved** (novox/hq to-be 45
+	// §8): the engine's witness judges the new build and restores the kept one when it is not healthy
+	// in bound. Placing decides what happens to the directory first — the running build moved aside,
+	// a build on trial discarded, a build rolled back here refused, the running build kept when it is
+	// the one declared — and keeps that, so a restoration later knows which build is where.
+	by := witnessOf(r)
+	placing := witness.Placing{Unpack: true}
+	if by == witness.ByNone {
+		// Nothing judged. Whatever was kept for a process that was judged before goes.
+		if err := witness.Forget(daemonRoot, r.Name); err != nil {
+			return out, err
+		}
+		// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
+		// previous version would be loaded by a runtime that walks a directory.
+		if err := os.RemoveAll(at); err != nil {
+			return out, err
+		}
+	} else {
+		placing, err = witness.Place(daemonRoot, r.Name, by, got, digestWritten(previous.Wrote), r.NotReversible, time.Now())
+		if err != nil {
+			return out, err
+		}
 	}
-	if err := os.MkdirAll(at, 0o755); err != nil {
-		return out, err
+	written := 0
+	if placing.Unpack {
+		if err := os.MkdirAll(at, 0o755); err != nil {
+			return out, err
+		}
+		if written, err = unpack(body, at); err != nil {
+			return out, err
+		}
+		if err := ownAll(at, r.User); err != nil {
+			return out, err
+		}
 	}
-	written, err := unpack(body, at)
-	if err != nil {
-		return out, err
-	}
-	if err := ownAll(at, r.User); err != nil {
-		return out, err
+	if placing.Commit != nil {
+		// Kept whatever the start below does: a build that would not start is still the build placed,
+		// and it is the witness's to judge.
+		defer func() {
+			if err := placing.Commit(); err != nil {
+				fmt.Fprintf(os.Stderr, "mesh-host: placed %s, and what the witness keeps about it could not be saved: %v\n",
+					r.Name, err)
+			}
+		}()
 	}

 	// **A step is run to completion, not installed.** What follows it is gated on it finishing,
@@ -201,9 +233,15 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		out.Action = "created"
 	}
 	out.Detail = fmt.Sprintf("%d file(s), running as %s.service", written, r.Name)
+	if !placing.Unpack {
+		out.Detail = fmt.Sprintf("its build already in place, running as %s.service", r.Name)
+	}
 	if because != "" {
 		out.Detail += ", restarted because " + because + " changed"
 	}
+	if placing.Detail != "" {
+		out.Detail += "; " + placing.Detail
+	}
 	out.wrote = want
 	return out, nil
 }
@@ -358,6 +396,13 @@ func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, st
 		}
 	}
 	bundle := filepath.Join(daemonRoot, name)
+	// And the build kept before it, with what the witness knew (novox/hq to-be 45 §8).
+	if _, err := os.Stat(witness.Dir(daemonRoot, name)); err == nil {
+		found = true
+		if err := witness.Forget(daemonRoot, name); err != nil {
+			return "", "", err
+		}
+	}
 	if _, err := os.Stat(bundle); err == nil {
 		found = true
 		if err := os.RemoveAll(bundle); err != nil {
@@ -381,3 +426,27 @@ func runFrom(r *declaration.Process) []string {
 	}
 	return run
 }
+
+// witnessOf is how a process's new builds are judged: as it declares, or by its name's default — the
+// mesh's two core processes (novox/hq to-be 45 §8). A step or a scheduled run is never judged.
+func witnessOf(r *declaration.Process) string {
+	if r.RunOnce || r.Schedule != "" {
+		return witness.ByNone
+	}
+	if r.Witness != "" {
+		return r.Witness
+	}
+	return witness.Default(r.Name)
+}
+
+// digestWritten is the bundle digest a process's record says was placed: the first half of what it
+// wrote, "sha256:<hex> <unit>".
+func digestWritten(wrote string) string {
+	if fields := strings.Fields(wrote); len(fields) > 0 && strings.HasPrefix(fields[0], "sha256:") {
+		return fields[0]
+	}
+	return ""
+}
+
+// DaemonRoot is where unpacked daemons live — and what a witness keeps beside them.
+func DaemonRoot() string { return daemonRoot }
@@ -0,0 +1,105 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/witness"
+)
+
+// A core process keeps the build before it while the new one is judged (novox/hq to-be 45 §8): the
+// applier places the new build where the unit runs it from, and moves the old one aside rather than
+// deleting it. A process nobody judges is replaced as ever.
+func TestACoreProcessKeepsTheBuildBeforeItWhileTheNewOneIsJudged(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+	run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
+
+	oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"})
+	newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"})
+	runtime := func(body []byte, digest string) string {
+		return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
+			`","digest":"` + digest + `","run":["./node-tools"]}`
+	}
+
+	_, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest)), store.State{},
+		store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(witness.Trials(bundles)) != 0 {
+		t.Fatal("a first placement went on trial with nothing to go back to")
+	}
+	_, _, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest)), state,
+		store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "new\n" {
+		t.Fatalf("the new build is not where the unit runs it from: %q", got)
+	}
+	if got, _ := os.ReadFile(filepath.Join(witness.Dir(bundles, "node-tools"), "previous", "node-tools")); string(got) != "old\n" {
+		t.Fatalf("the build before was not kept beside it: %q", got)
+	}
+	trials := witness.Trials(bundles)
+	if len(trials) != 1 || trials[0].Running != newDigest || trials[0].Previous != oldDigest || trials[0].Witness != witness.ByPing {
+		t.Fatalf("the new build is not on trial against the old one: %+v", trials)
+	}
+	// Undeclared: everything kept about it goes with it.
+	if _, _, err := removeProcess(context.Background(), store.Applied{Target: "node-tools"}, run); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(witness.Dir(bundles, "node-tools")); !os.IsNotExist(err) {
+		t.Fatalf("what the witness kept outlived the process: %v", err)
+	}
+}
+
+// A build rolled back on this machine is not placed again while the mesh still declares it; the
+// restored build keeps running, and the apply says why.
+func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+	run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
+
+	oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"})
+	newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"})
+	controller := func(body []byte, digest, env string) string {
+		return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) +
+			`","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}`
+	}
+	_, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{},
+		store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state,
+		store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run),
+		time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	// The mesh still declares the new build, and its unit changes: the process is re-placed.
+	report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state,
+		store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" {
+		t.Fatalf("the rolled-back build was placed again: %q", got)
+	}
+	if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") {
+		t.Fatalf("the apply does not say why it kept the build before: %+v", o)
+	}
+}