mesh-host/internal/apply/maintenance_window_test.go

package apply

import (
	"context"
	"errors"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

// A scheduled step may hold its module's own containers still while it runs (novox/hq ADR 0189,
// issue 108).
//
// What it exists for: the artifact store's collector walks the storage and requires every writer
// stopped. A run-once step runs beside containers and a scheduled one is the same container again,
// so the mesh had no way to say it — which is why the store it inherited has never collected
// anything. The risk the field brings is one shape only: a window that opens and never closes.
// Every test here is about that shape.

// windowRun records the order of stop / run / start, which is the whole of what is being asserted.
type windowRun struct {
	mu     sync.Mutex
	order  []string
	failAt string // the arg[0] that should fail ("run" makes the step fail)
	wontGo string // a container name that refuses to start again
}

func (w *windowRun) run(_ context.Context, _ string, args ...string) (string, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	switch args[0] {
	case "info":
		return "27.0\n", nil
	case "stop", "start":
		w.order = append(w.order, args[0]+" "+args[1])
		if args[0] == "start" && args[1] == w.wontGo {
			return "", errors.New("the runtime refused")
		}
	case "run":
		w.order = append(w.order, "run")
		if w.failAt == "run" {
			return "", errors.New("the step exited non-zero")
		}
	}
	return "", nil
}

func (w *windowRun) seen() []string {
	w.mu.Lock()
	defer w.mu.Unlock()
	return append([]string{}, w.order...)
}

// aStoreWithACollector is a module in the shape distribution has: a server that must not be
// writing, and a nightly step that walks its storage with the server held still.
func aStoreWithACollector(t *testing.T) *declaration.Declaration {
	t.Helper()
	return parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
	  {"id":"collect","type":"container","name":"mesh-registry-collect","image":"`+pinned+`",
	   "schedule":"30 3 * * *","while-stopped":["store"]}
	]}`)
}

func fireOnce(t *testing.T, d *declaration.Declaration, w *windowRun) {
	t.Helper()
	clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
	s := NewScheduler(clock, w.run, func(string) {})
	s.Sync(d, nil)
	s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
	s.Wait()
}

func TestAScheduledStepHoldsItsModulesContainerStillAndStartsItAgain(t *testing.T) {
	w := &windowRun{}
	fireOnce(t, aStoreWithACollector(t), w)

	got := w.seen()
	want := []string{"stop mesh-registry", "run", "start mesh-registry"}
	var kept []string
	for _, line := range got {
		if strings.HasPrefix(line, "stop mesh-registry-collect") {
			// Clearing the step's own exited container by name; not part of the window.
			continue
		}
		kept = append(kept, line)
	}
	if len(kept) != len(want) {
		t.Fatalf("the window was not stop, run, start: %v", got)
	}
	for i := range want {
		if kept[i] != want[i] {
			t.Fatalf("the window was %v, want %v", kept, want)
		}
	}
}

// The one that matters: a step that fails must leave the service running.
func TestAFailedStepStillClosesTheWindow(t *testing.T) {
	w := &windowRun{failAt: "run"}
	fireOnce(t, aStoreWithACollector(t), w)

	var started bool
	for _, line := range w.seen() {
		if line == "start mesh-registry" {
			started = true
		}
	}
	if !started {
		t.Fatalf("the step failed and the container it held still was never started again: %v", w.seen())
	}
}

// A container that will not come back is said loudly: it is down, and nothing else notices until
// the next apply compares it.
func TestAContainerThatWillNotStartAgainIsSaidLoudly(t *testing.T) {
	w := &windowRun{wontGo: "mesh-registry"}
	var said []string
	clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
	s := NewScheduler(clock, w.run, func(line string) { said = append(said, line) })
	s.Sync(aStoreWithACollector(t), nil)
	s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
	s.Wait()

	var loud bool
	for _, line := range said {
		if strings.Contains(line, "WILL NOT START AGAIN") && strings.Contains(line, "mesh-registry") {
			loud = true
		}
	}
	if !loud {
		t.Fatalf("a service left stopped by a maintenance window was not said loudly: %v", said)
	}
}

// Several containers come back in the reverse of the order they were stopped: a module names the
// dependant first, and starting it before what it depends on is not bringing it back.
func TestTheWindowClosesInTheReverseOfTheOrderItOpened(t *testing.T) {
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"web","type":"container","name":"web","image":"`+pinned+`"},
	  {"id":"db","type":"container","name":"db","image":"`+pinned+`"},
	  {"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
	   "schedule":"30 3 * * *","while-stopped":["web","db"]}
	]}`)
	w := &windowRun{}
	fireOnce(t, d, w)

	var stops, starts []string
	for _, line := range w.seen() {
		switch {
		case line == "stop web" || line == "stop db":
			stops = append(stops, line)
		case strings.HasPrefix(line, "start "):
			starts = append(starts, line)
		}
	}
	if len(stops) != 2 || stops[0] != "stop web" || stops[1] != "stop db" {
		t.Fatalf("stopped in %v, want the order the step named them", stops)
	}
	if len(starts) != 2 || starts[0] != "start db" || starts[1] != "start web" {
		t.Fatalf("started in %v, want the reverse", starts)
	}
}

// And the refusals, each for what it says rather than that it says something.
func TestAMaintenanceWindowIsRefusedWhereItCannotMean(t *testing.T) {
	for _, c := range []struct{ name, body, says string }{
		{
			"a window with no schedule",
			`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","while-stopped":["store"]}`,
			"needs a schedule",
		},
		{
			"a window naming itself",
			`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["collect"]}`,
			"this step itself",
		},
		{
			"a window naming something that is not a container here",
			`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["elsewhere"]}`,
			"no container by that id",
		},
	} {
		_, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` + c.body + `]}`))
		if err == nil {
			t.Errorf("%s was accepted", c.name)
			continue
		}
		if !strings.Contains(err.Error(), c.says) {
			t.Errorf("%s: the refusal does not say %q: %v", c.name, c.says, err)
		}
	}
}

// A changed window is a changed declaration, and the install says so.
//
// The cadence already works this way: "a changed schedule is a changed spec — the marker moves and
// the install is reported updated and re-established" (containerSpec). Which containers are held
// still for the run is the same kind of statement, and a declaration that changed it while the
// machine reported no change would be a machine quietly running the old window.
func TestAChangedWindowMovesTheSpec(t *testing.T) {
	one := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
	  {"id":"other","type":"container","name":"other","image":"`+pinned+`"},
	  {"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
	   "schedule":"30 3 * * *","while-stopped":["store"]}
	]}`)
	two := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
	  {"id":"other","type":"container","name":"other","image":"`+pinned+`"},
	  {"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
	   "schedule":"30 3 * * *","while-stopped":["store","other"]}
	]}`)
	stepOf := func(d *declaration.Declaration) *declaration.Container {
		for _, r := range d.Resources {
			if c, ok := r.(*declaration.Container); ok && c.ID == "collect" {
				return c
			}
		}
		t.Fatal("no step in the fixture")
		return nil
	}
	if containerSpec(stepOf(one), inputs{}) == containerSpec(stepOf(two), inputs{}) {
		t.Fatal("the window changed and the spec did not; the machine would report no change " +
			"and keep holding the containers it held yesterday")
	}
	// And a container with no window is untouched by the field existing at all.
	plain := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"}
	]}`)
	spec := containerSpec(plain.Resources[0].(*declaration.Container), inputs{})
	if strings.Contains(spec, "while-stopped") || strings.Contains(spec, "held") {
		t.Errorf("an ordinary container's spec mentions a field it does not set:\n%s", spec)
	}
}