jschoubben
b602b223e1
containerSpec says a changed cadence moves the marker so the install is reported updated and re-established. Which containers are held still is the same kind of statement, and a declaration that changed it while the machine reported no change would be a machine quietly holding yesterday's containers.
239 lines
8.6 KiB
Go
239 lines
8.6 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// A scheduled step may hold its module's own containers still while it runs (novox/hq ADR 0189,
|
|
// issue 108).
|
|
//
|
|
// What it exists for: the artifact store's collector walks the storage and requires every writer
|
|
// stopped. A run-once step runs beside containers and a scheduled one is the same container again,
|
|
// so the mesh had no way to say it — which is why the store it inherited has never collected
|
|
// anything. The risk the field brings is one shape only: a window that opens and never closes.
|
|
// Every test here is about that shape.
|
|
|
|
// windowRun records the order of stop / run / start, which is the whole of what is being asserted.
|
|
type windowRun struct {
|
|
mu sync.Mutex
|
|
order []string
|
|
failAt string // the arg[0] that should fail ("run" makes the step fail)
|
|
wontGo string // a container name that refuses to start again
|
|
}
|
|
|
|
func (w *windowRun) run(_ context.Context, _ string, args ...string) (string, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "stop", "start":
|
|
w.order = append(w.order, args[0]+" "+args[1])
|
|
if args[0] == "start" && args[1] == w.wontGo {
|
|
return "", errors.New("the runtime refused")
|
|
}
|
|
case "run":
|
|
w.order = append(w.order, "run")
|
|
if w.failAt == "run" {
|
|
return "", errors.New("the step exited non-zero")
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func (w *windowRun) seen() []string {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
return append([]string{}, w.order...)
|
|
}
|
|
|
|
// aStoreWithACollector is a module in the shape distribution has: a server that must not be
|
|
// writing, and a nightly step that walks its storage with the server held still.
|
|
func aStoreWithACollector(t *testing.T) *declaration.Declaration {
|
|
t.Helper()
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
|
{"id":"collect","type":"container","name":"mesh-registry-collect","image":"`+pinned+`",
|
|
"schedule":"30 3 * * *","while-stopped":["store"]}
|
|
]}`)
|
|
}
|
|
|
|
func fireOnce(t *testing.T, d *declaration.Declaration, w *windowRun) {
|
|
t.Helper()
|
|
clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
|
|
s := NewScheduler(clock, w.run, func(string) {})
|
|
s.Sync(d, nil)
|
|
s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
|
|
s.Wait()
|
|
}
|
|
|
|
func TestAScheduledStepHoldsItsModulesContainerStillAndStartsItAgain(t *testing.T) {
|
|
w := &windowRun{}
|
|
fireOnce(t, aStoreWithACollector(t), w)
|
|
|
|
got := w.seen()
|
|
want := []string{"stop mesh-registry", "run", "start mesh-registry"}
|
|
var kept []string
|
|
for _, line := range got {
|
|
if strings.HasPrefix(line, "stop mesh-registry-collect") {
|
|
// Clearing the step's own exited container by name; not part of the window.
|
|
continue
|
|
}
|
|
kept = append(kept, line)
|
|
}
|
|
if len(kept) != len(want) {
|
|
t.Fatalf("the window was not stop, run, start: %v", got)
|
|
}
|
|
for i := range want {
|
|
if kept[i] != want[i] {
|
|
t.Fatalf("the window was %v, want %v", kept, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The one that matters: a step that fails must leave the service running.
|
|
func TestAFailedStepStillClosesTheWindow(t *testing.T) {
|
|
w := &windowRun{failAt: "run"}
|
|
fireOnce(t, aStoreWithACollector(t), w)
|
|
|
|
var started bool
|
|
for _, line := range w.seen() {
|
|
if line == "start mesh-registry" {
|
|
started = true
|
|
}
|
|
}
|
|
if !started {
|
|
t.Fatalf("the step failed and the container it held still was never started again: %v", w.seen())
|
|
}
|
|
}
|
|
|
|
// A container that will not come back is said loudly: it is down, and nothing else notices until
|
|
// the next apply compares it.
|
|
func TestAContainerThatWillNotStartAgainIsSaidLoudly(t *testing.T) {
|
|
w := &windowRun{wontGo: "mesh-registry"}
|
|
var said []string
|
|
clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
|
|
s := NewScheduler(clock, w.run, func(line string) { said = append(said, line) })
|
|
s.Sync(aStoreWithACollector(t), nil)
|
|
s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
|
|
s.Wait()
|
|
|
|
var loud bool
|
|
for _, line := range said {
|
|
if strings.Contains(line, "WILL NOT START AGAIN") && strings.Contains(line, "mesh-registry") {
|
|
loud = true
|
|
}
|
|
}
|
|
if !loud {
|
|
t.Fatalf("a service left stopped by a maintenance window was not said loudly: %v", said)
|
|
}
|
|
}
|
|
|
|
// Several containers come back in the reverse of the order they were stopped: a module names the
|
|
// dependant first, and starting it before what it depends on is not bringing it back.
|
|
func TestTheWindowClosesInTheReverseOfTheOrderItOpened(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"web","type":"container","name":"web","image":"`+pinned+`"},
|
|
{"id":"db","type":"container","name":"db","image":"`+pinned+`"},
|
|
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
|
"schedule":"30 3 * * *","while-stopped":["web","db"]}
|
|
]}`)
|
|
w := &windowRun{}
|
|
fireOnce(t, d, w)
|
|
|
|
var stops, starts []string
|
|
for _, line := range w.seen() {
|
|
switch {
|
|
case line == "stop web" || line == "stop db":
|
|
stops = append(stops, line)
|
|
case strings.HasPrefix(line, "start "):
|
|
starts = append(starts, line)
|
|
}
|
|
}
|
|
if len(stops) != 2 || stops[0] != "stop web" || stops[1] != "stop db" {
|
|
t.Fatalf("stopped in %v, want the order the step named them", stops)
|
|
}
|
|
if len(starts) != 2 || starts[0] != "start db" || starts[1] != "start web" {
|
|
t.Fatalf("started in %v, want the reverse", starts)
|
|
}
|
|
}
|
|
|
|
// And the refusals, each for what it says rather than that it says something.
|
|
func TestAMaintenanceWindowIsRefusedWhereItCannotMean(t *testing.T) {
|
|
for _, c := range []struct{ name, body, says string }{
|
|
{
|
|
"a window with no schedule",
|
|
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","while-stopped":["store"]}`,
|
|
"needs a schedule",
|
|
},
|
|
{
|
|
"a window naming itself",
|
|
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["collect"]}`,
|
|
"this step itself",
|
|
},
|
|
{
|
|
"a window naming something that is not a container here",
|
|
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["elsewhere"]}`,
|
|
"no container by that id",
|
|
},
|
|
} {
|
|
_, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` + c.body + `]}`))
|
|
if err == nil {
|
|
t.Errorf("%s was accepted", c.name)
|
|
continue
|
|
}
|
|
if !strings.Contains(err.Error(), c.says) {
|
|
t.Errorf("%s: the refusal does not say %q: %v", c.name, c.says, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A changed window is a changed declaration, and the install says so.
|
|
//
|
|
// The cadence already works this way: "a changed schedule is a changed spec — the marker moves and
|
|
// the install is reported updated and re-established" (containerSpec). Which containers are held
|
|
// still for the run is the same kind of statement, and a declaration that changed it while the
|
|
// machine reported no change would be a machine quietly running the old window.
|
|
func TestAChangedWindowMovesTheSpec(t *testing.T) {
|
|
one := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
|
{"id":"other","type":"container","name":"other","image":"`+pinned+`"},
|
|
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
|
"schedule":"30 3 * * *","while-stopped":["store"]}
|
|
]}`)
|
|
two := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
|
{"id":"other","type":"container","name":"other","image":"`+pinned+`"},
|
|
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
|
"schedule":"30 3 * * *","while-stopped":["store","other"]}
|
|
]}`)
|
|
stepOf := func(d *declaration.Declaration) *declaration.Container {
|
|
for _, r := range d.Resources {
|
|
if c, ok := r.(*declaration.Container); ok && c.ID == "collect" {
|
|
return c
|
|
}
|
|
}
|
|
t.Fatal("no step in the fixture")
|
|
return nil
|
|
}
|
|
if containerSpec(stepOf(one), inputs{}) == containerSpec(stepOf(two), inputs{}) {
|
|
t.Fatal("the window changed and the spec did not; the machine would report no change " +
|
|
"and keep holding the containers it held yesterday")
|
|
}
|
|
// And a container with no window is untouched by the field existing at all.
|
|
plain := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"}
|
|
]}`)
|
|
spec := containerSpec(plain.Resources[0].(*declaration.Container), inputs{})
|
|
if strings.Contains(spec, "while-stopped") || strings.Contains(spec, "held") {
|
|
t.Errorf("an ordinary container's spec mentions a field it does not set:\n%s", spec)
|
|
}
|
|
}
|