Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b602b223e1 | ||
|
|
e8c4824ae2 | ||
|
|
8c2e76f4c1 | ||
|
|
8280a82ef8 | ||
|
|
2d5e76434b | ||
|
|
d9ea387680 | ||
|
|
99ad145bec |
+22
-2
@@ -1458,6 +1458,15 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
// undo is never reported as done; not fatal for a former target, which was never dropped by anyone.
|
||||
var errNoRemoval = errors.New("no way to remove")
|
||||
|
||||
// exited is a command that ran and exited non-zero: its words, and the exit itself.
|
||||
type exited struct {
|
||||
words string
|
||||
exit *exec.ExitError
|
||||
}
|
||||
|
||||
func (e *exited) Error() string { return e.words }
|
||||
func (e *exited) Unwrap() error { return e.exit }
|
||||
|
||||
// ExecRunner runs a real command, with stdin closed and output captured.
|
||||
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
@@ -1466,8 +1475,12 @@ func ExecRunner(ctx context.Context, name string, args ...string) (string, error
|
||||
if err != nil {
|
||||
var exit *exec.ExitError
|
||||
if errors.As(err, &exit) {
|
||||
return string(out), fmt.Errorf("%s exited %d: %s",
|
||||
name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
||||
// The words as they always were, and the exit underneath them, so a caller asks the
|
||||
// code (system.ExitCode) rather than matching text that this line is free to reword.
|
||||
return string(out), &exited{
|
||||
words: fmt.Sprintf("%s exited %d: %s", name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))),
|
||||
exit: exit,
|
||||
}
|
||||
}
|
||||
return string(out), fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
@@ -1694,6 +1707,13 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
|
||||
// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
|
||||
// moves and the install is reported "updated" and re-established. Added only when present, so no
|
||||
// ordinary container's or run-once step's digest moves for a field it does not set.
|
||||
// And which of its module's containers it holds still while it runs (novox/hq ADR 0189), for
|
||||
// the same reason: a declaration that changed the window while the machine reported no change
|
||||
// would be a machine quietly holding yesterday's containers. In declared order, which is the
|
||||
// order they are stopped in.
|
||||
for _, id := range r.WhileStopped {
|
||||
b.WriteString("while-stopped " + id + "\n")
|
||||
}
|
||||
if r.Schedule != "" {
|
||||
b.WriteString("schedule " + r.Schedule + "\n")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// A scheduled step may hold its module's own containers still while it runs (novox/hq ADR 0189,
|
||||
// issue 108).
|
||||
//
|
||||
// What it exists for: the artifact store's collector walks the storage and requires every writer
|
||||
// stopped. A run-once step runs beside containers and a scheduled one is the same container again,
|
||||
// so the mesh had no way to say it — which is why the store it inherited has never collected
|
||||
// anything. The risk the field brings is one shape only: a window that opens and never closes.
|
||||
// Every test here is about that shape.
|
||||
|
||||
// windowRun records the order of stop / run / start, which is the whole of what is being asserted.
|
||||
type windowRun struct {
|
||||
mu sync.Mutex
|
||||
order []string
|
||||
failAt string // the arg[0] that should fail ("run" makes the step fail)
|
||||
wontGo string // a container name that refuses to start again
|
||||
}
|
||||
|
||||
func (w *windowRun) run(_ context.Context, _ string, args ...string) (string, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch args[0] {
|
||||
case "info":
|
||||
return "27.0\n", nil
|
||||
case "stop", "start":
|
||||
w.order = append(w.order, args[0]+" "+args[1])
|
||||
if args[0] == "start" && args[1] == w.wontGo {
|
||||
return "", errors.New("the runtime refused")
|
||||
}
|
||||
case "run":
|
||||
w.order = append(w.order, "run")
|
||||
if w.failAt == "run" {
|
||||
return "", errors.New("the step exited non-zero")
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (w *windowRun) seen() []string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return append([]string{}, w.order...)
|
||||
}
|
||||
|
||||
// aStoreWithACollector is a module in the shape distribution has: a server that must not be
|
||||
// writing, and a nightly step that walks its storage with the server held still.
|
||||
func aStoreWithACollector(t *testing.T) *declaration.Declaration {
|
||||
t.Helper()
|
||||
return parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
||||
{"id":"collect","type":"container","name":"mesh-registry-collect","image":"`+pinned+`",
|
||||
"schedule":"30 3 * * *","while-stopped":["store"]}
|
||||
]}`)
|
||||
}
|
||||
|
||||
func fireOnce(t *testing.T, d *declaration.Declaration, w *windowRun) {
|
||||
t.Helper()
|
||||
clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
|
||||
s := NewScheduler(clock, w.run, func(string) {})
|
||||
s.Sync(d, nil)
|
||||
s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
|
||||
s.Wait()
|
||||
}
|
||||
|
||||
func TestAScheduledStepHoldsItsModulesContainerStillAndStartsItAgain(t *testing.T) {
|
||||
w := &windowRun{}
|
||||
fireOnce(t, aStoreWithACollector(t), w)
|
||||
|
||||
got := w.seen()
|
||||
want := []string{"stop mesh-registry", "run", "start mesh-registry"}
|
||||
var kept []string
|
||||
for _, line := range got {
|
||||
if strings.HasPrefix(line, "stop mesh-registry-collect") {
|
||||
// Clearing the step's own exited container by name; not part of the window.
|
||||
continue
|
||||
}
|
||||
kept = append(kept, line)
|
||||
}
|
||||
if len(kept) != len(want) {
|
||||
t.Fatalf("the window was not stop, run, start: %v", got)
|
||||
}
|
||||
for i := range want {
|
||||
if kept[i] != want[i] {
|
||||
t.Fatalf("the window was %v, want %v", kept, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The one that matters: a step that fails must leave the service running.
|
||||
func TestAFailedStepStillClosesTheWindow(t *testing.T) {
|
||||
w := &windowRun{failAt: "run"}
|
||||
fireOnce(t, aStoreWithACollector(t), w)
|
||||
|
||||
var started bool
|
||||
for _, line := range w.seen() {
|
||||
if line == "start mesh-registry" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
if !started {
|
||||
t.Fatalf("the step failed and the container it held still was never started again: %v", w.seen())
|
||||
}
|
||||
}
|
||||
|
||||
// A container that will not come back is said loudly: it is down, and nothing else notices until
|
||||
// the next apply compares it.
|
||||
func TestAContainerThatWillNotStartAgainIsSaidLoudly(t *testing.T) {
|
||||
w := &windowRun{wontGo: "mesh-registry"}
|
||||
var said []string
|
||||
clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
|
||||
s := NewScheduler(clock, w.run, func(line string) { said = append(said, line) })
|
||||
s.Sync(aStoreWithACollector(t), nil)
|
||||
s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
|
||||
s.Wait()
|
||||
|
||||
var loud bool
|
||||
for _, line := range said {
|
||||
if strings.Contains(line, "WILL NOT START AGAIN") && strings.Contains(line, "mesh-registry") {
|
||||
loud = true
|
||||
}
|
||||
}
|
||||
if !loud {
|
||||
t.Fatalf("a service left stopped by a maintenance window was not said loudly: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// Several containers come back in the reverse of the order they were stopped: a module names the
|
||||
// dependant first, and starting it before what it depends on is not bringing it back.
|
||||
func TestTheWindowClosesInTheReverseOfTheOrderItOpened(t *testing.T) {
|
||||
d := parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"web","type":"container","name":"web","image":"`+pinned+`"},
|
||||
{"id":"db","type":"container","name":"db","image":"`+pinned+`"},
|
||||
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
||||
"schedule":"30 3 * * *","while-stopped":["web","db"]}
|
||||
]}`)
|
||||
w := &windowRun{}
|
||||
fireOnce(t, d, w)
|
||||
|
||||
var stops, starts []string
|
||||
for _, line := range w.seen() {
|
||||
switch {
|
||||
case line == "stop web" || line == "stop db":
|
||||
stops = append(stops, line)
|
||||
case strings.HasPrefix(line, "start "):
|
||||
starts = append(starts, line)
|
||||
}
|
||||
}
|
||||
if len(stops) != 2 || stops[0] != "stop web" || stops[1] != "stop db" {
|
||||
t.Fatalf("stopped in %v, want the order the step named them", stops)
|
||||
}
|
||||
if len(starts) != 2 || starts[0] != "start db" || starts[1] != "start web" {
|
||||
t.Fatalf("started in %v, want the reverse", starts)
|
||||
}
|
||||
}
|
||||
|
||||
// And the refusals, each for what it says rather than that it says something.
|
||||
func TestAMaintenanceWindowIsRefusedWhereItCannotMean(t *testing.T) {
|
||||
for _, c := range []struct{ name, body, says string }{
|
||||
{
|
||||
"a window with no schedule",
|
||||
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","while-stopped":["store"]}`,
|
||||
"needs a schedule",
|
||||
},
|
||||
{
|
||||
"a window naming itself",
|
||||
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["collect"]}`,
|
||||
"this step itself",
|
||||
},
|
||||
{
|
||||
"a window naming something that is not a container here",
|
||||
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["elsewhere"]}`,
|
||||
"no container by that id",
|
||||
},
|
||||
} {
|
||||
_, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` + c.body + `]}`))
|
||||
if err == nil {
|
||||
t.Errorf("%s was accepted", c.name)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: the refusal does not say %q: %v", c.name, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A changed window is a changed declaration, and the install says so.
|
||||
//
|
||||
// The cadence already works this way: "a changed schedule is a changed spec — the marker moves and
|
||||
// the install is reported updated and re-established" (containerSpec). Which containers are held
|
||||
// still for the run is the same kind of statement, and a declaration that changed it while the
|
||||
// machine reported no change would be a machine quietly running the old window.
|
||||
func TestAChangedWindowMovesTheSpec(t *testing.T) {
|
||||
one := parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
||||
{"id":"other","type":"container","name":"other","image":"`+pinned+`"},
|
||||
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
||||
"schedule":"30 3 * * *","while-stopped":["store"]}
|
||||
]}`)
|
||||
two := parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
||||
{"id":"other","type":"container","name":"other","image":"`+pinned+`"},
|
||||
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
||||
"schedule":"30 3 * * *","while-stopped":["store","other"]}
|
||||
]}`)
|
||||
stepOf := func(d *declaration.Declaration) *declaration.Container {
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok && c.ID == "collect" {
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatal("no step in the fixture")
|
||||
return nil
|
||||
}
|
||||
if containerSpec(stepOf(one), inputs{}) == containerSpec(stepOf(two), inputs{}) {
|
||||
t.Fatal("the window changed and the spec did not; the machine would report no change " +
|
||||
"and keep holding the containers it held yesterday")
|
||||
}
|
||||
// And a container with no window is untouched by the field existing at all.
|
||||
plain := parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"}
|
||||
]}`)
|
||||
spec := containerSpec(plain.Resources[0].(*declaration.Container), inputs{})
|
||||
if strings.Contains(spec, "while-stopped") || strings.Contains(spec, "held") {
|
||||
t.Errorf("an ordinary container's spec mentions a field it does not set:\n%s", spec)
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,29 @@ type scheduledJob struct {
|
||||
container *declaration.Container
|
||||
next time.Time // the next minute at which it is due
|
||||
running bool // a run is in flight — the next due run is skipped rather than stacked
|
||||
// hold is the runtime names of the containers held still for the duration of a run, in the
|
||||
// order the step named them (novox/hq ADR 0189).
|
||||
hold []string
|
||||
}
|
||||
|
||||
// heldStillFor is the runtime names of the containers a step holds still, resolved from ids.
|
||||
func heldStillFor(step *declaration.Container, d *declaration.Declaration) []string {
|
||||
if len(step.WhileStopped) == 0 {
|
||||
return nil
|
||||
}
|
||||
byID := map[string]string{}
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok {
|
||||
byID[c.Identity()] = c.Name
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(step.WhileStopped))
|
||||
for _, id := range step.WhileStopped {
|
||||
if name := byID[id]; name != "" {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewScheduler builds a scheduler. A nil clock is the system clock; a nil log says nothing.
|
||||
@@ -123,15 +146,20 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
|
||||
// on its cadence rather than staying running to be restarted — so its identity cannot
|
||||
// depend on another resource's content and there is nothing to pass.
|
||||
spec := containerSpec(c, inputs{})
|
||||
// The runtime stops containers by name; the declaration names them by id. Resolved here,
|
||||
// against the declaration this job was armed from, so a fire never has to look anything up
|
||||
// (novox/hq ADR 0189). The parser has already refused an id that is not a container here.
|
||||
hold := heldStillFor(c, d)
|
||||
if existing := s.jobs[c.Identity()]; existing != nil && existing.spec == spec {
|
||||
// Unchanged: keep where it is in its cadence, refresh the declaration pointer only.
|
||||
existing.container = c
|
||||
existing.hold = hold
|
||||
continue
|
||||
}
|
||||
// New or changed: arm it for the next due minute after now.
|
||||
next, _ := cron.Next(s.clock.Now())
|
||||
s.jobs[c.Identity()] = &scheduledJob{
|
||||
id: c.Identity(), spec: spec, cron: cron, container: c, next: next,
|
||||
id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +230,15 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) {
|
||||
return
|
||||
}
|
||||
|
||||
// **The window opens here and closes in the defer, whatever happens** (novox/hq ADR 0189).
|
||||
// Deferred before the first stop so a panic, a failing step or a step that runs long all end
|
||||
// the same way: the service running. The one real risk of this field is a window that never
|
||||
// closes, and the only defence against it is that closing is not conditional on anything.
|
||||
if len(j.hold) > 0 {
|
||||
defer s.letRun(ctx, cri, j)
|
||||
s.holdStill(ctx, cri, j)
|
||||
}
|
||||
|
||||
// A container by this name left exited by the previous run would collide with --name. Removing
|
||||
// one that is not there is the state we want, so its error is ignored — the same as run-once.
|
||||
_, _ = s.run(ctx, cri, "rm", "-f", j.container.Name)
|
||||
@@ -260,3 +297,51 @@ func (s *Scheduler) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// holdStill stops the containers this step runs instead of, in the order it named them.
|
||||
//
|
||||
// A stop that fails is said and not fatal. The step runs anyway: for the case this exists for —
|
||||
// a collector walking storage nothing must be writing to — a writer that would not stop is worth
|
||||
// knowing about, and refusing to run would mean the work never happens and the log says nothing
|
||||
// new each night. What must not be skipped is the restart, and it is not: it is deferred.
|
||||
func (s *Scheduler) holdStill(ctx context.Context, cri string, j *scheduledJob) {
|
||||
for _, name := range j.hold {
|
||||
if _, err := s.run(ctx, cri, "stop", name); err != nil {
|
||||
s.log(fmt.Sprintf("scheduled step %s: could not stop %s for the run: %v", j.id, name, err))
|
||||
continue
|
||||
}
|
||||
s.log(fmt.Sprintf("scheduled step %s: %s held still for the run", j.id, name))
|
||||
}
|
||||
}
|
||||
|
||||
// letRun starts them again, in the reverse of the order they were stopped, and says so loudly if
|
||||
// one does not come back.
|
||||
//
|
||||
// **Reverse order**, because stopping walks a dependency the other way: a module that holds two
|
||||
// containers still names the one that depends on the other first, and bringing them back the same
|
||||
// way would start a dependant before what it depends on.
|
||||
//
|
||||
// Given its own context, because this runs in a defer and the one the run used may already be
|
||||
// cancelled — a host shutting down mid-window would otherwise leave the service stopped, which is
|
||||
// precisely the outcome this field must never have.
|
||||
func (s *Scheduler) letRun(_ context.Context, cri string, j *scheduledJob) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), closingWindow)
|
||||
defer cancel()
|
||||
for i := len(j.hold) - 1; i >= 0; i-- {
|
||||
name := j.hold[i]
|
||||
if _, err := s.run(ctx, cri, "start", name); err != nil {
|
||||
// Said as loudly as this host says anything: a service the mesh stopped for a
|
||||
// maintenance window and could not start again is down, and nothing else will notice
|
||||
// until the next apply compares it.
|
||||
s.log(fmt.Sprintf(
|
||||
"scheduled step %s: %s was held still for the run and WILL NOT START AGAIN: %v",
|
||||
j.id, name, err))
|
||||
continue
|
||||
}
|
||||
s.log(fmt.Sprintf("scheduled step %s: %s running again", j.id, name))
|
||||
}
|
||||
}
|
||||
|
||||
// closingWindow is how long the host will spend putting back what it stopped. Generous: this is
|
||||
// the half that must not be given up on.
|
||||
const closingWindow = 5 * time.Minute
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package apply
|
||||
|
||||
// ForTests points where the host writes units and unpacks daemons at directories a test owns, and
|
||||
// waits nothing between its looks at a unit, until the returned function puts them back. For tests
|
||||
// in other packages that apply a process — the bootstrap's, which checks that what genesis raises is
|
||||
// what the controller's process takes over (novox/hq issue 223). Nothing outside a test calls it.
|
||||
func ForTests(units, daemons string) (restore func()) {
|
||||
wasUnits, wasDaemons, wasSettle, wasHandover := unitDir, daemonRoot, serviceSettle, handoverSettle
|
||||
unitDir, daemonRoot, serviceSettle, handoverSettle = units, daemons, 0, 0
|
||||
return func() {
|
||||
unitDir, daemonRoot, serviceSettle, handoverSettle = wasUnits, wasDaemons, wasSettle, wasHandover
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -112,6 +114,43 @@ func BuildControlPlane(ctx context.Context, run Runner, builderTag string, sourc
|
||||
return Built{}, nil
|
||||
}
|
||||
|
||||
// **Which form the controller is in at that commit** (novox/hq issue 223). An image the module
|
||||
// builds is the form genesis always raised, and the builder builds it as before. A process the
|
||||
// module runs from a Go bundle cannot be built here — its toolchain is one the mesh makes later —
|
||||
// so genesis builds the controller's own Dockerfile and raises it as the container that process
|
||||
// replaces (genesis_form.go).
|
||||
workspace, err := os.MkdirTemp("", "mesh-genesis-*")
|
||||
if err != nil {
|
||||
return Built{}, err
|
||||
}
|
||||
defer os.RemoveAll(workspace)
|
||||
dir, commit, err := cloneAt(ctx, run, builderTag, source, workspace)
|
||||
if err != nil {
|
||||
return Built{}, fmt.Errorf("the control plane could not be fetched from %s at %s: %w",
|
||||
source.Repository, shortRef(source.Ref), err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
return Built{}, fmt.Errorf("%s at %s has no module manifest: %w", source.Repository, shortRef(source.Ref), err)
|
||||
}
|
||||
form, err := processFormOf(raw)
|
||||
if err != nil {
|
||||
return Built{}, err
|
||||
}
|
||||
if form.Found {
|
||||
image, err := buildGenesisImage(ctx, run, dir)
|
||||
if err != nil {
|
||||
return Built{}, err
|
||||
}
|
||||
manifest, err := genesisForm(raw, form, image)
|
||||
if err != nil {
|
||||
return Built{}, err
|
||||
}
|
||||
say(fmt.Sprintf(" built %s from %s, as the container its process %s replaces (%s)",
|
||||
ControlPlaneModule, shortRef(commit), form.Process, form.Replaces))
|
||||
return Built{Module: ControlPlaneModule, Commit: commit, Image: image, Manifest: manifest}, nil
|
||||
}
|
||||
|
||||
out, err := run(ctx, "docker", args...)
|
||||
if err != nil {
|
||||
return Built{}, fmt.Errorf("the control plane could not be built from %s at %s: %w",
|
||||
|
||||
@@ -49,10 +49,10 @@ func TestARepositoryAndACommitIsEnough(t *testing.T) {
|
||||
func TestTheBuildHandsOverTheManifestTheMeshWillHold(t *testing.T) {
|
||||
manifest := `{"module":"mesh-controller","version":"1","resources":[` +
|
||||
`{"id":"server","type":"container","name":"mesh-controller","image":"` + builtImage + `"}]}`
|
||||
runtime := &asked{answer: func(string, []string) (string, error) {
|
||||
runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
|
||||
return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":` + manifest +
|
||||
`,"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}` + "\n", nil
|
||||
}}
|
||||
})}
|
||||
built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
|
||||
Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
|
||||
if err != nil {
|
||||
@@ -69,10 +69,10 @@ func TestTheBuildHandsOverTheManifestTheMeshWillHold(t *testing.T) {
|
||||
// A result without a manifest is a build the installer cannot finish, and it is refused beside the
|
||||
// builder that said it rather than at step 9 with a message about a missing file.
|
||||
func TestABuildReportingNoManifestIsRefused(t *testing.T) {
|
||||
runtime := &asked{answer: func(string, []string) (string, error) {
|
||||
runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
|
||||
return `{"module":"mesh-controller","commit":"a1b2c3d4",` +
|
||||
`"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
|
||||
}}
|
||||
})}
|
||||
_, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
|
||||
Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
|
||||
if err == nil || !strings.Contains(err.Error(), "no manifest") {
|
||||
@@ -82,11 +82,11 @@ func TestABuildReportingNoManifestIsRefused(t *testing.T) {
|
||||
|
||||
// And a manifest that does not name the image the build produced describes some other build.
|
||||
func TestABuildWhoseManifestNamesAnotherImageIsRefused(t *testing.T) {
|
||||
runtime := &asked{answer: func(string, []string) (string, error) {
|
||||
runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
|
||||
return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":{"module":"mesh-controller",` +
|
||||
`"resources":[{"id":"server","type":"container","image":"sha256:` + strings.Repeat("9", 64) + `"}]},` +
|
||||
`"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
|
||||
}}
|
||||
})}
|
||||
_, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
|
||||
Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not name that image") {
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The controller as a process, raised at genesis as a container (novox/hq issue 213, issue 223).
|
||||
//
|
||||
// **The mesh runs the controller as a Go bundle the host starts as a process; genesis cannot.** A
|
||||
// process's bundle is fetched from the mesh's artifact store, which genesis raises long after the
|
||||
// controller, and a Go bundle is compiled in a toolchain the mesh builds later still. So genesis
|
||||
// pivots to the controller as it always has — an image it built from the controller's own
|
||||
// Dockerfile, run as a container the temporary controller composes — and the first time the mesh
|
||||
// builds the controller from its repository, the controller's own declaration is the process, which
|
||||
// names that container under `replaces`, and the host hands over: the process is started, seen up,
|
||||
// and only then is the container removed (mesh-host `replaces`, issue 213).
|
||||
//
|
||||
// **The container is genesis's shape, not the manifest's.** The manifest declares only the process.
|
||||
// From it genesis takes what the process is given — its environment, which is host paths and words —
|
||||
// and the id the process replaces; the container around it is written here: the image genesis built,
|
||||
// the host's network, and every host path the environment names mounted at the same path read-only.
|
||||
// It runs as the image's own unprivileged user (65534), so the secrets belong to that number until
|
||||
// the process's account takes them over — the image is FROM scratch and knows no account by name. Its
|
||||
// id is the one the process replaces, so the first declaration the mesh composes for this machine
|
||||
// hands this container over rather than leaving two controllers running.
|
||||
|
||||
// genesisUser is who the genesis container runs as — the image's own USER — and who its secrets
|
||||
// belong to until the process takes them over: the image has no passwd to look an account up in.
|
||||
const genesisUser = "65534:65534"
|
||||
|
||||
// ProcessForm is the controller's process, as its manifest declares it, and the container id that
|
||||
// process replaces. Found is false for a manifest in the image form — an older controller — which
|
||||
// genesis installs as it always did.
|
||||
type ProcessForm struct {
|
||||
Found bool
|
||||
Process string // the process resource's id
|
||||
Replaces string // the id of the container genesis raises in its place
|
||||
}
|
||||
|
||||
// processFormOf finds the controller's process in its manifest: a process resource running a bundle
|
||||
// the module builds, saying which one resource it replaces.
|
||||
func processFormOf(manifest []byte) (ProcessForm, error) {
|
||||
var m struct {
|
||||
Build *struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return ProcessForm{}, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
kinds[a.Name] = a.Kind
|
||||
}
|
||||
}
|
||||
for _, kind := range kinds {
|
||||
if kind == "image" {
|
||||
return ProcessForm{}, nil // the image form: the builder builds it, as before
|
||||
}
|
||||
}
|
||||
var found []ProcessForm
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "process" || kinds[fmt.Sprint(r["artifact"])] != "bundle" {
|
||||
continue
|
||||
}
|
||||
if once, _ := r["run-once"].(bool); once {
|
||||
continue
|
||||
}
|
||||
replaces, _ := r["replaces"].([]any)
|
||||
if len(replaces) != 1 {
|
||||
return ProcessForm{}, fmt.Errorf(
|
||||
"the %s module runs as the process %v and says it replaces %v. Genesis raises the "+
|
||||
"controller as a container that process takes over, so the process names exactly "+
|
||||
"one resource it replaces — the id genesis gives the container",
|
||||
ControlPlaneModule, r["id"], r["replaces"])
|
||||
}
|
||||
found = append(found, ProcessForm{Found: true, Process: fmt.Sprint(r["id"]),
|
||||
Replaces: fmt.Sprint(replaces[0])})
|
||||
}
|
||||
if len(found) != 1 {
|
||||
return ProcessForm{}, fmt.Errorf(
|
||||
"the %s module builds no image and runs %d process(es) of its own; genesis raises one "+
|
||||
"controller, from the process its manifest declares", ControlPlaneModule, len(found))
|
||||
}
|
||||
return found[0], nil
|
||||
}
|
||||
|
||||
// genesisForm is the manifest genesis registers: the controller's own manifest, its process
|
||||
// replaced by the container genesis runs in its place, under the id the process replaces, running
|
||||
// the image genesis built. Resolved as a build would resolve it — no build section, the image named
|
||||
// — because that is what the temporary controller is handed.
|
||||
func genesisForm(manifest []byte, form ProcessForm, image string) ([]byte, error) {
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resources, _ := m["resources"].([]any)
|
||||
var out []any
|
||||
for _, raw := range resources {
|
||||
r, _ := raw.(map[string]any)
|
||||
if r == nil || r["id"] != form.Process {
|
||||
out = append(out, raw)
|
||||
continue
|
||||
}
|
||||
env, _ := r["env"].(map[string]any)
|
||||
var volumes []any
|
||||
for _, key := range sortedAnyKeys(env) {
|
||||
value := fmt.Sprint(env[key])
|
||||
if strings.HasPrefix(value, "/") || strings.HasPrefix(value, "${dir:") {
|
||||
volumes = append(volumes, value+":"+value+":ro")
|
||||
}
|
||||
}
|
||||
container := map[string]any{
|
||||
"id": form.Replaces, "type": "container", "name": ControlPlaneModule,
|
||||
"image": image, "network": "host", "args": []any{"serve"},
|
||||
}
|
||||
if len(env) > 0 {
|
||||
container["env"] = env
|
||||
}
|
||||
if len(volumes) > 0 {
|
||||
container["volumes"] = volumes
|
||||
}
|
||||
out = append(out, container)
|
||||
}
|
||||
m["resources"] = out
|
||||
// What the container reads must be readable by who it runs as. The process's account owns them
|
||||
// once the process takes over, and the host gives them to it in the same apply.
|
||||
m["secrets-owner"] = genesisUser
|
||||
// **Nothing to prepare at genesis.** The temporary controller — the same commit — migrated the
|
||||
// stores when the foundation raised it, and a preparation step is derived from a resource running
|
||||
// an artifact the module built, which a pinned image is not: the controller refuses `prepares`
|
||||
// with nothing to run it in. The process prepares the stores itself when it takes over.
|
||||
delete(m, "prepares")
|
||||
delete(m, "build")
|
||||
var b bytes.Buffer
|
||||
enc := json.NewEncoder(&b)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.TrimSpace(b.Bytes()), nil
|
||||
}
|
||||
|
||||
func sortedAnyKeys(m map[string]any) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// cloneAt fetches the controller's repository at the commit genesis builds, into a directory of
|
||||
// this machine's, with the carried builder's git — the machine is not assumed to have one. Returns
|
||||
// the module's directory and the commit that was checked out.
|
||||
func cloneAt(ctx context.Context, run Runner, builderTag string, source Source, into string) (string, string, error) {
|
||||
git := func(args ...string) (string, error) {
|
||||
return run(ctx, "docker", append([]string{"run", "--rm", "-v", into + ":/ws",
|
||||
"--entrypoint", "git", builderTag}, args...)...)
|
||||
}
|
||||
if _, err := git("clone", "--quiet", source.Repository, "/ws/src"); err != nil {
|
||||
return "", "", fmt.Errorf("cloning %s: %w", source.Repository, err)
|
||||
}
|
||||
if _, err := git("-C", "/ws/src", "checkout", "--quiet", "--detach", source.Ref); err != nil {
|
||||
return "", "", fmt.Errorf("checking out %s: %w", shortRef(source.Ref), err)
|
||||
}
|
||||
commit, err := git("-C", "/ws/src", "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return filepath.Join(into, "src", source.Path), strings.TrimSpace(commit), nil
|
||||
}
|
||||
|
||||
// buildGenesisImage builds the controller's image from its own Dockerfile (the one `make image`
|
||||
// uses), on this machine, and names it by the digest of its own configuration, as the builder did.
|
||||
func buildGenesisImage(ctx context.Context, run Runner, dir string) (string, error) {
|
||||
if _, err := os.Stat(filepath.Join(dir, "Dockerfile")); err != nil {
|
||||
return "", fmt.Errorf("the %s repository has no Dockerfile, so genesis has no image to raise "+
|
||||
"the controller from: %w", ControlPlaneModule, err)
|
||||
}
|
||||
out, err := run(ctx, "docker", "build", "--quiet", dir)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("building the %s image: %w", ControlPlaneModule, err)
|
||||
}
|
||||
image := strings.TrimSpace(out)
|
||||
if i := strings.LastIndex(image, "\n"); i >= 0 {
|
||||
image = strings.TrimSpace(image[i+1:])
|
||||
}
|
||||
if !strings.HasPrefix(image, "sha256:") {
|
||||
return "", fmt.Errorf("docker build said %q, which is not an image id", firstLine(out))
|
||||
}
|
||||
return image, nil
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// imageFormManifest is a controller from before issue 213: it builds an image and runs a container.
|
||||
const imageFormManifest = `{"module":"mesh-controller","version":"1","resources":[
|
||||
{"id":"server","type":"container","name":"mesh-controller","network":"host","artifact":"server"}],
|
||||
"build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}}`
|
||||
|
||||
// processFormManifest is the controller's manifest as novox/mesh-controller declares it after issue
|
||||
// 213: a Go bundle the host runs as a process, replacing the container it ran as.
|
||||
const processFormManifest = `{
|
||||
"module": "mesh-controller", "version": "1", "slug": "control", "prepares": true,
|
||||
"claims": [{"name": "mesh-controller", "scope": "mesh"}],
|
||||
"accesses": [{"path": "/var/lib/mesh-broker-tls", "mode": "read"}],
|
||||
"own-secrets": {"inventory": "${dir:mesh-state}/inventory", "bus": "${dir:mesh-state}/bus"},
|
||||
"secrets-owner": "mesh-controller",
|
||||
"tools": ["status"],
|
||||
"resources": [
|
||||
{"id": "account", "type": "user", "name": "mesh-controller", "shell": "/usr/bin/nologin", "home": "/var/lib/mesh-controller"},
|
||||
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh", "owner": "mesh-controller"},
|
||||
{"id": "controller", "type": "process", "name": "mesh-controller", "artifact": "controller",
|
||||
"run": ["./mesh-controller", "serve"], "user": "mesh-controller",
|
||||
"env": {"MESH_BROKER_CERTIFICATE": "/var/lib/mesh-broker-tls/tls.crt",
|
||||
"MESH_STORE_INVENTORY_FILE": "${dir:mesh-state}/inventory",
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"},
|
||||
"replaces": ["server"]}
|
||||
],
|
||||
"build": {"artifacts": [{"name": "controller", "kind": "bundle", "language": "go", "system": "arch",
|
||||
"from": "cmd/mesh-controller", "binary": "mesh-controller"}]}
|
||||
}`
|
||||
|
||||
// aRepository answers the carried builder's git as a clone of a repository holding this manifest and
|
||||
// a Dockerfile, and hands every other command to then.
|
||||
func aRepository(t *testing.T, manifest string, then func(string, []string) (string, error)) func(string, []string) (string, error) {
|
||||
t.Helper()
|
||||
return func(name string, args []string) (string, error) {
|
||||
if name == "docker" && len(args) > 5 && args[0] == "run" && args[4] == "--entrypoint" && args[5] == "git" {
|
||||
host := strings.TrimSuffix(args[3], ":/ws")
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, " clone "):
|
||||
src := filepath.Join(host, "src")
|
||||
if err := os.MkdirAll(src, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(src, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", os.WriteFile(filepath.Join(src, "Dockerfile"), []byte("FROM scratch\n"), 0o644)
|
||||
case strings.Contains(joined, "rev-parse"):
|
||||
return "a1b2c3d4e5f6\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
return then(name, args)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 223: a controller declared as a process is raised at genesis as a container built
|
||||
// from its own Dockerfile, not by the builder — which has no Go toolchain at genesis and would refuse.
|
||||
func TestAProcessFormControllerIsBuiltFromItsDockerfileAndRaisedAsAContainer(t *testing.T) {
|
||||
runtime := &asked{answer: aRepository(t, processFormManifest, func(name string, args []string) (string, error) {
|
||||
if name == "docker" && args[0] == "build" {
|
||||
return builtImage + "\n", nil
|
||||
}
|
||||
t.Fatalf("genesis ran %s %v; a process-form controller is built from its Dockerfile alone", name, args)
|
||||
return "", nil
|
||||
})}
|
||||
built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
|
||||
Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if built.Image != builtImage || built.Commit != "a1b2c3d4e5f6" {
|
||||
t.Errorf("built %q from %q", built.Image, built.Commit)
|
||||
}
|
||||
if runtime.ran("mesh-builder:test build") {
|
||||
t.Error("the builder was asked to build a controller it cannot build at genesis")
|
||||
}
|
||||
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(built.Manifest, &m); err != nil {
|
||||
t.Fatalf("the genesis manifest is not JSON: %v", err)
|
||||
}
|
||||
if _, has := m["prepares"]; has {
|
||||
t.Error("the genesis manifest prepares its state; the temporary controller already did, and a pinned image is nothing the controller can derive a step from")
|
||||
}
|
||||
if _, has := m["build"]; has {
|
||||
t.Error("the genesis manifest still says how it is built; it is handed over resolved")
|
||||
}
|
||||
var container map[string]any
|
||||
for _, raw := range m["resources"].([]any) {
|
||||
r := raw.(map[string]any)
|
||||
if r["type"] == "process" {
|
||||
t.Errorf("the genesis manifest still runs the process: %v", r)
|
||||
}
|
||||
if r["type"] == "container" {
|
||||
container = r
|
||||
}
|
||||
}
|
||||
if container == nil {
|
||||
t.Fatal("the genesis manifest runs no container")
|
||||
}
|
||||
for key, want := range map[string]any{"id": "server", "name": "mesh-controller", "image": builtImage,
|
||||
"network": "host"} {
|
||||
if container[key] != want {
|
||||
t.Errorf("the genesis container's %s is %v, not %v", key, container[key], want)
|
||||
}
|
||||
}
|
||||
if _, has := container["user"]; has {
|
||||
t.Error("the genesis container says a user; the host's container has no such field, the image's USER is who it runs as")
|
||||
}
|
||||
if m["secrets-owner"] != "65534:65534" {
|
||||
t.Errorf("the secrets belong to %v, which the container cannot read as", m["secrets-owner"])
|
||||
}
|
||||
volumes, _ := json.Marshal(container["volumes"])
|
||||
for _, want := range []string{"${dir:mesh-state}/inventory:${dir:mesh-state}/inventory:ro",
|
||||
"/var/lib/mesh-broker-tls/tls.crt:/var/lib/mesh-broker-tls/tls.crt:ro"} {
|
||||
if !strings.Contains(string(volumes), want) {
|
||||
t.Errorf("the container does not mount %s: %s", want, volumes)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(volumes), "seat:") {
|
||||
t.Errorf("a word that is not a path was mounted: %s", volumes)
|
||||
}
|
||||
|
||||
// And it is what step 9 installs: pinned, its container found, its stores delivered from its
|
||||
// environment — the same path an image-form controller takes.
|
||||
pinned, _, err := pinImage(built.Manifest, built.Image, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id := controlPlaneResourceIn(pinned); id != "server" {
|
||||
t.Errorf("step 9 finds the controller's container as %q", id)
|
||||
}
|
||||
wanted, err := secretsByVariableIn(pinned)
|
||||
if err != nil {
|
||||
t.Fatalf("step 9 cannot deliver the stores into the genesis container: %v", err)
|
||||
}
|
||||
if wanted["MESH_STORE_INVENTORY"] != "inventory" {
|
||||
t.Errorf("step 9 delivers %v", wanted)
|
||||
}
|
||||
}
|
||||
|
||||
// An older controller — an image and a container — is still built by the builder, as before.
|
||||
func TestAnImageFormControllerIsStillBuiltByTheBuilder(t *testing.T) {
|
||||
manifest := `{"module":"mesh-controller","version":"1","resources":[` +
|
||||
`{"id":"server","type":"container","name":"mesh-controller","image":"` + builtImage + `"}]}`
|
||||
runtime := &asked{answer: aRepository(t, imageFormManifest, func(name string, args []string) (string, error) {
|
||||
if name == "docker" && args[0] == "build" {
|
||||
t.Fatal("an image-form controller was built from its Dockerfile rather than by the builder")
|
||||
}
|
||||
return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":` + manifest +
|
||||
`,"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
|
||||
})}
|
||||
built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
|
||||
Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(built.Manifest) != manifest || !runtime.ran("mesh-builder:test build") {
|
||||
t.Errorf("the image form did not go through the builder: %s", built.Manifest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) {
|
||||
for _, bad := range []string{`"replaces": []`, `"replaces": ["a", "b"]`} {
|
||||
raw := strings.Replace(processFormManifest, `"replaces": ["server"]`, bad, 1)
|
||||
if _, err := processFormOf([]byte(raw)); err == nil {
|
||||
t.Errorf("a process saying %s was accepted; genesis would not know what to name its container", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// novox/hq issue 223: what genesis raises is what the controller's process takes over. The temporary
|
||||
// controller composes the genesis container, and the host records it as `<module>.<its id>`; the
|
||||
// first declaration the mesh composes from the controller's real manifest names that same id under
|
||||
// the process's `replaces` (the composer prefixes both alike — mesh-controller's
|
||||
// TestTheControllerIsAProcessAndNoContainer). So the first apply hands over: the process is started,
|
||||
// seen up, and only then is the genesis container removed — one controller before, one after, never
|
||||
// none and never two left.
|
||||
func TestTheFirstApplyHandsTheGenesisContainerOverToTheProcess(t *testing.T) {
|
||||
restore := apply.ForTests(t.TempDir(), t.TempDir())
|
||||
defer restore()
|
||||
|
||||
form, err := processFormOf([]byte(processFormManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
genesis, err := genesisForm([]byte(processFormManifest), form, builtImage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What the host records for the genesis container, as the composer names it.
|
||||
recorded := ""
|
||||
var m struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(genesis, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "container" {
|
||||
recorded = ControlPlaneModule + "." + r["id"].(string)
|
||||
}
|
||||
}
|
||||
known := store.State{Resources: []store.Applied{{Origin: store.OriginDeclared, ID: recorded,
|
||||
Type: "container", Target: ControlPlaneModule}}}
|
||||
|
||||
// The controller's first composed declaration: its process, replacing what the manifest names.
|
||||
body, digest := aBundle(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }))
|
||||
defer server.Close()
|
||||
replaces, _ := json.Marshal([]string{ControlPlaneModule + "." + form.Replaces})
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"` + ControlPlaneModule + "." + form.Process +
|
||||
`","type":"process","name":"mesh-controller","source":"` + server.URL + `/c.tgz","digest":"` + digest +
|
||||
`","run":["./mesh-controller","serve"],"replaces":` + string(replaces) + `}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var commands []string
|
||||
started, container := false, true
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
commands = append(commands, line)
|
||||
switch {
|
||||
case strings.HasPrefix(line, "systemctl restart mesh-controller.service"):
|
||||
started = true
|
||||
case strings.HasPrefix(line, "systemctl show mesh-controller.service") && started:
|
||||
return "ActiveState=active\nSubState=running\nMainPID=7\nNRestarts=0\n", nil
|
||||
case strings.HasPrefix(line, "docker rm -f mesh-controller"):
|
||||
if !started {
|
||||
t.Error("the genesis container was removed before the process was started")
|
||||
}
|
||||
container = false
|
||||
case strings.HasPrefix(line, "docker container inspect") && !container:
|
||||
return "", errors.New("no such container")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
sys, err := system.For("arch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, after, err := apply.Apply(context.Background(), sys, d, known, store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the first apply did not hand over: %v\n%s", err, strings.Join(commands, "\n"))
|
||||
}
|
||||
if container {
|
||||
t.Fatalf("the genesis container is still running beside the process — two controllers:\n%s",
|
||||
strings.Join(commands, "\n"))
|
||||
}
|
||||
if _, still := after.Find(recorded); still {
|
||||
t.Error("the host still records the genesis container")
|
||||
}
|
||||
}
|
||||
|
||||
func aBundle(t *testing.T) ([]byte, string) {
|
||||
t.Helper()
|
||||
var raw bytes.Buffer
|
||||
zipped := gzip.NewWriter(&raw)
|
||||
w := tar.NewWriter(zipped)
|
||||
content := "#!/bin/sh\n"
|
||||
if err := w.WriteHeader(&tar.Header{Name: "mesh-controller", Mode: 0o755, Size: int64(len(content)), Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = w.Write([]byte(content))
|
||||
_ = w.Close()
|
||||
_ = zipped.Close()
|
||||
sum := sha256.Sum256(raw.Bytes())
|
||||
return raw.Bytes(), "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -1037,6 +1037,24 @@ type Container struct {
|
||||
// rather than stacked. It is exclusive with RunOnce and with restart-on: a container runs once
|
||||
// and gates, runs on a cadence, or stays up — never two of these.
|
||||
Schedule string `json:"schedule,omitempty"`
|
||||
|
||||
// WhileStopped names resources of the same module — containers — that must be held still for
|
||||
// the duration of this step's run (novox/hq ADR 0189). The host stops each before the run and
|
||||
// starts each again after it, **whatever the step did**: a step that failed must leave the
|
||||
// service running, because the one real risk of this field is a window that never closes.
|
||||
//
|
||||
// **For the work a service cannot have done underneath it.** The artifact store's collector
|
||||
// walks the storage and requires every writer stopped; a run-once step runs beside containers
|
||||
// and a scheduled one is the same container again, so until this there was no way for a module
|
||||
// to say it. The predecessor said it with a shell script, which is how the mesh inherited a
|
||||
// store that has never collected anything.
|
||||
//
|
||||
// **Its own module's containers, and only on a schedule.** A module that could quiesce a
|
||||
// neighbour could stop the mesh. And at apply time the host already has a window — the
|
||||
// declaration is applied in order and a run-once step gates what follows — so a one-time
|
||||
// offline job says *before*, not *instead of*; a recurring window is the case order cannot
|
||||
// express, and the only one this serves.
|
||||
WhileStopped []string `json:"while-stopped,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Container) Identity() string { return c.ID }
|
||||
@@ -1068,6 +1086,20 @@ func (c *Container) validate(where string, _ bool) []string {
|
||||
problems = append(problems, where+": "+err.Error())
|
||||
}
|
||||
}
|
||||
// A maintenance window belongs to a recurring step (novox/hq ADR 0189). Refused on anything
|
||||
// else here, where the field is; that it names containers of the same module, and not itself,
|
||||
// is judged against the whole declaration (see whileStoppedNames).
|
||||
if len(c.WhileStopped) > 0 && c.Schedule == "" {
|
||||
problems = append(problems, where+": while-stopped needs a schedule; at apply the host "+
|
||||
"already has a window — the declaration is applied in order and a run-once step gates "+
|
||||
"what follows — so a one-time offline job is declared before what it works on")
|
||||
}
|
||||
for _, id := range c.WhileStopped {
|
||||
if id == c.ID {
|
||||
problems = append(problems, where+": while-stopped names "+strconv.Quote(id)+
|
||||
", which is this step itself")
|
||||
}
|
||||
}
|
||||
// The runtime's flags take addresses, and a name here would be handed to it verbatim and
|
||||
// refused at create — after the old container was already removed. Refused on arrival instead.
|
||||
for _, d := range c.Dns {
|
||||
@@ -1482,6 +1514,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
problems = append(problems, resource.validate(where, allowActions)...)
|
||||
d.Resources = append(d.Resources, resource)
|
||||
}
|
||||
problems = append(problems, checkWhileStopped(d.Resources)...)
|
||||
problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
|
||||
problems = append(problems, checkReplaces(d.Resources)...)
|
||||
if env.Adoption == nil {
|
||||
@@ -1511,6 +1544,41 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// checkWhileStopped judges a maintenance window against the whole declaration (novox/hq ADR 0189).
|
||||
//
|
||||
// A step may hold still only a container that is **here** — in this same declaration, which is to
|
||||
// say on this machine and placed by the mesh. That is what makes it the module's own: a node's
|
||||
// declaration carries one module's resources beside another's, so the id must also be a container
|
||||
// and not a file or a directory, which there would be nothing to stop.
|
||||
//
|
||||
// Refused on arrival rather than discovered at the first fire. A window that names something the
|
||||
// host cannot stop is a window that opens at 03:00 and reports nothing until somebody reads a log.
|
||||
func checkWhileStopped(resources []Resource) []string {
|
||||
containers := map[string]bool{}
|
||||
for _, r := range resources {
|
||||
if r.Kind() == TypeContainer {
|
||||
containers[r.Identity()] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, r := range resources {
|
||||
c, ok := r.(*Container)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, id := range c.WhileStopped {
|
||||
if containers[id] {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"resource %q: while-stopped names %q, and this declaration has no container by "+
|
||||
"that id. A step may hold still only a container placed on this machine "+
|
||||
"beside it", c.ID, id))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func strictDecode(raw []byte, into any) error {
|
||||
// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
|
||||
// field the host does not know is a thing the control plane believes it asked for.
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A user that does not exist yet is "absent", in the words the host's own runner uses
|
||||
// ("getent exited 2"), not only Go's ("exit status 2"). Matched as text, the runner's wording read as
|
||||
// a user database that did not answer, and the controller's account was never created.
|
||||
func TestAMissingUserIsAbsentInTheRunnersWords(t *testing.T) {
|
||||
for _, words := range []string{"getent exited 2: ", "exit status 2"} {
|
||||
run := func(context.Context, string, ...string) (string, error) { return "", errors.New(words) }
|
||||
_, found, err := LookUpUser(context.Background(), run, "nobody-here")
|
||||
if err != nil || found {
|
||||
t.Errorf("%q: found %v, err %v; want absent", words, found, err)
|
||||
}
|
||||
}
|
||||
run := func(context.Context, string, ...string) (string, error) { return "", errors.New("getent exited 1: ") }
|
||||
if _, _, err := LookUpUser(context.Background(), run, "x"); err == nil {
|
||||
t.Error("a database that failed read as an answer")
|
||||
}
|
||||
}
|
||||
|
||||
// The real command, through a real exit: getent's code for a key not found.
|
||||
func TestAMissingUserIsAbsentFromTheRealGetent(t *testing.T) {
|
||||
if _, err := exec.LookPath("getent"); err != nil {
|
||||
t.Skip("no getent here")
|
||||
}
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
out, err := exec.CommandContext(ctx, name, args...).Output()
|
||||
return string(out), err
|
||||
}
|
||||
_, found, err := LookUpUser(context.Background(), run, "mesh-no-such-user-0b1f")
|
||||
if err != nil || found {
|
||||
t.Errorf("found %v, err %v; want absent", found, err)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
@@ -95,8 +98,10 @@ func LookUpUser(ctx context.Context, run Runner, name string) (Login, bool, erro
|
||||
out, err := run(ctx, "getent", "passwd", name)
|
||||
if err != nil {
|
||||
// getent's own convention: 2 means the key was not found, which is the only failure that
|
||||
// means "no such user".
|
||||
if strings.Contains(err.Error(), "exit status 2") {
|
||||
// means "no such user". Asked of the exit code: matched as text, it looked for Go's wording
|
||||
// ("exit status 2") while the host's runner says "getent exited 2", so a user that did not
|
||||
// exist yet read as a database that did not answer, and no account was ever created.
|
||||
if code, ok := ExitCode(err); ok && code == 2 {
|
||||
return Login{}, false, nil
|
||||
}
|
||||
return Login{}, false, fmt.Errorf(
|
||||
@@ -221,3 +226,27 @@ func For(name string) (System, error) {
|
||||
func All() []System {
|
||||
return []System{arch{}, alpine{}, android{}}
|
||||
}
|
||||
|
||||
// ExitCode is the code a command exited with, when err says one: from the exit itself where the
|
||||
// runner kept it, else from the words either runner shape uses ("exit status N", "<cmd> exited N").
|
||||
func ExitCode(err error) (int, bool) {
|
||||
if err == nil {
|
||||
return 0, false
|
||||
}
|
||||
var exit *exec.ExitError
|
||||
if errors.As(err, &exit) {
|
||||
return exit.ExitCode(), true
|
||||
}
|
||||
if m := exitWords.FindStringSubmatch(err.Error()); len(m) == 3 {
|
||||
for _, g := range m[1:] {
|
||||
if g != "" {
|
||||
if n, convErr := strconv.Atoi(g); convErr == nil {
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var exitWords = regexp.MustCompile(`exit status (\d+)|exited (\d+)`)
|
||||
|
||||
Reference in New Issue
Block a user