mesh-host/internal/apply/process_witness_test.go

package apply

import (
	"context"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/witness"
)

// A core process keeps the build before it while the new one is judged (novox/hq to-be 45 §8): the
// applier places the new build where the unit runs it from, and moves the old one aside rather than
// deleting it. A process nobody judges is replaced as ever.
func TestACoreProcessKeepsTheBuildBeforeItWhileTheNewOneIsJudged(t *testing.T) {
	units, bundles := t.TempDir(), t.TempDir()
	wasUnits, wasBundles := unitDir, daemonRoot
	unitDir, daemonRoot = units, bundles
	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
	run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }

	oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"})
	newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"})
	runtime := func(body []byte, digest string) string {
		return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
			`","digest":"` + digest + `","run":["./node-tools"]}`
	}

	_, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest)), store.State{},
		store.OriginDeclared, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if len(witness.Trials(bundles)) != 0 {
		t.Fatal("a first placement went on trial with nothing to go back to")
	}
	_, _, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest)), state,
		store.OriginDeclared, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "new\n" {
		t.Fatalf("the new build is not where the unit runs it from: %q", got)
	}
	if got, _ := os.ReadFile(filepath.Join(witness.Dir(bundles, "node-tools"), "previous", "node-tools")); string(got) != "old\n" {
		t.Fatalf("the build before was not kept beside it: %q", got)
	}
	trials := witness.Trials(bundles)
	if len(trials) != 1 || trials[0].Running != newDigest || trials[0].Previous != oldDigest || trials[0].Witness != witness.ByPing {
		t.Fatalf("the new build is not on trial against the old one: %+v", trials)
	}
	// Undeclared: everything kept about it goes with it.
	if _, _, err := removeProcess(context.Background(), store.Applied{Target: "node-tools"}, run); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(witness.Dir(bundles, "node-tools")); !os.IsNotExist(err) {
		t.Fatalf("what the witness kept outlived the process: %v", err)
	}
}

// A build rolled back on this machine is not placed again while the mesh still declares it; the
// restored build keeps running, and the apply says why.
func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
	units, bundles := t.TempDir(), t.TempDir()
	wasUnits, wasBundles := unitDir, daemonRoot
	unitDir, daemonRoot = units, bundles
	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
	run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }

	oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"})
	newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"})
	controller := func(body []byte, digest, env string) string {
		return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) +
			`","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}`
	}
	_, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{},
		store.OriginDeclared, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	_, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state,
		store.OriginDeclared, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run),
		time.Now()); err != nil {
		t.Fatal(err)
	}
	// The mesh still declares the new build, and its unit changes: the process is re-placed.
	report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state,
		store.OriginDeclared, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" {
		t.Fatalf("the rolled-back build was placed again: %q", got)
	}
	if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") {
		t.Fatalf("the apply does not say why it kept the build before: %+v", o)
	}
}

// **A process nobody judges keeps the directory it works in until its restart** (novox/hq issue 734).
// On 2026-10-26 three checks on one builder were refused in the second a new build of it arrived: the
// applier deleted the running build's directory before the restart, and the `go version` the old
// process started in it failed with "getwd: no such file or directory". Here a process works in the
// build's directory without running from it, as that `go version` did: the apply of a new build leaves
// its directory in place, and the retired build goes once nothing works in it.
func TestAProcessNobodyJudgesKeepsItsDirectoryUntilItsRestart(t *testing.T) {
	if _, err := os.Stat("/proc/self/cwd"); err != nil {
		t.Skip("which process works in a build is read from /proc")
	}
	units, bundles := t.TempDir(), t.TempDir()
	wasUnits, wasBundles := unitDir, daemonRoot
	unitDir, daemonRoot = units, bundles
	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
	run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }

	oldBody, oldDigest := anArchive(t, map[string]string{"mesh-build-go": "old\n"})
	newBody, newDigest := anArchive(t, map[string]string{"mesh-build-go": "new\n"})
	builder := func(body []byte, digest string) string {
		return `{"id":"mesh-build-go.builder","type":"process","name":"mesh-build-go","source":"` + serving(t, body) +
			`","digest":"` + digest + `","run":["./mesh-build-go"]}`
	}
	_, state, err := Apply(context.Background(), archHost(t), declare(t, builder(oldBody, oldDigest)), store.State{},
		store.OriginDeclared, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	sleep, err := exec.LookPath("sleep")
	if err != nil {
		t.Skip("no sleep to run")
	}
	working := exec.Command(sleep, "60")
	working.Dir = filepath.Join(bundles, "mesh-build-go")
	if err := working.Start(); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = working.Process.Kill(); _ = working.Wait() })
	cwd := filepath.Join("/proc", strconv.Itoa(working.Process.Pid), "cwd")
	for deadline := time.Now().Add(5 * time.Second); ; {
		if at, err := os.Readlink(cwd); err == nil && at == working.Dir {
			break
		}
		if time.Now().After(deadline) {
			t.Fatalf("PID %d does not work in %s within 5s", working.Process.Pid, working.Dir)
		}
		time.Sleep(10 * time.Millisecond)
	}

	if _, _, err := Apply(context.Background(), archHost(t), declare(t, builder(newBody, newDigest)), state,
		store.OriginDeclared, run, nil, nil); err != nil {
		t.Fatal(err)
	}
	if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-build-go", "mesh-build-go")); string(got) != "new\n" {
		t.Fatalf("the new build is not where the unit runs it from: %q", got)
	}
	at, err := os.Readlink(cwd)
	if err != nil || strings.HasSuffix(at, " (deleted)") {
		t.Fatalf("the directory a running process works in was deleted before its restart: %q (%v)", at, err)
	}
	if got, _ := os.ReadFile(filepath.Join(at, "mesh-build-go")); string(got) != "old\n" {
		t.Fatalf("the process's directory no longer holds its own build: %q", got)
	}
	if len(witness.Trials(bundles)) != 0 {
		t.Fatal("a process nobody judges went on trial")
	}

	// Its process gone, the next look deletes the retired build and keeps nothing about the process.
	_ = working.Process.Kill()
	_ = working.Wait()
	witness.SweepAll(bundles)
	if _, err := os.Stat(witness.Dir(bundles, "mesh-build-go")); !os.IsNotExist(err) {
		t.Fatalf("a retired build nothing works in was kept: %v", err)
	}
}