The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
106 lines
4.7 KiB
Go
106 lines
4.7 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/store"
|
|
"github.com/novox/mesh-host/internal/witness"
|
|
)
|
|
|
|
// A core process keeps the build before it while the new one is judged (novox/hq to-be 45 §8): the
|
|
// applier places the new build where the unit runs it from, and moves the old one aside rather than
|
|
// deleting it. A process nobody judges is replaced as ever.
|
|
func TestACoreProcessKeepsTheBuildBeforeItWhileTheNewOneIsJudged(t *testing.T) {
|
|
units, bundles := t.TempDir(), t.TempDir()
|
|
wasUnits, wasBundles := unitDir, daemonRoot
|
|
unitDir, daemonRoot = units, bundles
|
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
|
|
|
oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"})
|
|
newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"})
|
|
runtime := func(body []byte, digest string) string {
|
|
return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
|
|
`","digest":"` + digest + `","run":["./node-tools"]}`
|
|
}
|
|
|
|
_, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest)), store.State{},
|
|
store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(witness.Trials(bundles)) != 0 {
|
|
t.Fatal("a first placement went on trial with nothing to go back to")
|
|
}
|
|
_, _, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest)), state,
|
|
store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "new\n" {
|
|
t.Fatalf("the new build is not where the unit runs it from: %q", got)
|
|
}
|
|
if got, _ := os.ReadFile(filepath.Join(witness.Dir(bundles, "node-tools"), "previous", "node-tools")); string(got) != "old\n" {
|
|
t.Fatalf("the build before was not kept beside it: %q", got)
|
|
}
|
|
trials := witness.Trials(bundles)
|
|
if len(trials) != 1 || trials[0].Running != newDigest || trials[0].Previous != oldDigest || trials[0].Witness != witness.ByPing {
|
|
t.Fatalf("the new build is not on trial against the old one: %+v", trials)
|
|
}
|
|
// Undeclared: everything kept about it goes with it.
|
|
if _, _, err := removeProcess(context.Background(), store.Applied{Target: "node-tools"}, run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(witness.Dir(bundles, "node-tools")); !os.IsNotExist(err) {
|
|
t.Fatalf("what the witness kept outlived the process: %v", err)
|
|
}
|
|
}
|
|
|
|
// A build rolled back on this machine is not placed again while the mesh still declares it; the
|
|
// restored build keeps running, and the apply says why.
|
|
func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
|
|
units, bundles := t.TempDir(), t.TempDir()
|
|
wasUnits, wasBundles := unitDir, daemonRoot
|
|
unitDir, daemonRoot = units, bundles
|
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
|
|
|
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"})
|
|
newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"})
|
|
controller := func(body []byte, digest, env string) string {
|
|
return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) +
|
|
`","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}`
|
|
}
|
|
_, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{},
|
|
store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state,
|
|
store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run),
|
|
time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The mesh still declares the new build, and its unit changes: the process is re-placed.
|
|
report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state,
|
|
store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" {
|
|
t.Fatalf("the rolled-back build was placed again: %q", got)
|
|
}
|
|
if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") {
|
|
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
|
}
|
|
}
|