Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f3e135dfc9 |
@@ -58,8 +58,6 @@ type Outcome struct {
|
||||
kept string
|
||||
// stateless is a service whose unit's lifecycle is the machine's (novox/hq ADR 0117).
|
||||
stateless bool
|
||||
// scope and user are, for a service, whose manager it was applied through (novox/hq ADR 0177).
|
||||
scope, user string
|
||||
// found is, for a service, its unit as the host first found it (novox/hq ADR 0118).
|
||||
found *store.FoundUnit
|
||||
// reads is, for a container, the digest of each file it was created reading, by path — so
|
||||
@@ -565,8 +563,6 @@ func ApplyKeeping(
|
||||
Kept: kept,
|
||||
Reads: outcome.reads,
|
||||
Stateless: outcome.stateless,
|
||||
Scope: outcome.scope,
|
||||
User: outcome.user,
|
||||
Found: outcome.found,
|
||||
Holds: holds(resource),
|
||||
})
|
||||
@@ -1047,12 +1043,10 @@ type unitReloader interface {
|
||||
|
||||
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
||||
changed map[string]bool, previous store.Applied) (Outcome, error) {
|
||||
run = managerFor(r.Scope, r.User, run)
|
||||
if r.Stateless() {
|
||||
return reflectOnly(ctx, sys, r, run, changed)
|
||||
}
|
||||
out := begin(r)
|
||||
out.scope, out.user = r.Scope, r.User
|
||||
var changes []string
|
||||
|
||||
// A file the service reflects changed, and it may be the unit's own file or a drop-in: the
|
||||
@@ -1192,9 +1186,7 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
||||
// a machine that uses another — and it reads the change when whatever starts it does.
|
||||
func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
||||
changed map[string]bool) (Outcome, error) {
|
||||
run = managerFor(r.Scope, r.User, run)
|
||||
out := begin(r)
|
||||
out.scope, out.user = r.Scope, r.User
|
||||
out.stateless = true
|
||||
restart := reflected(r, changed)
|
||||
reload := restartedBy(r.ReloadOn, changed)
|
||||
@@ -1415,25 +1407,6 @@ func applyPackage(ctx context.Context, sys system.System, r *declaration.Package
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if r.Absent {
|
||||
// Declared absent (novox/hq ADR 0175): removed when it is here, left alone when it is not.
|
||||
if !installed {
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "not installed, as declared"
|
||||
return out, nil
|
||||
}
|
||||
if err := sys.RemovePackage(ctx, run, r.Package); err != nil {
|
||||
return out, fmt.Errorf("removing %s: %w", r.Package, err)
|
||||
}
|
||||
if still, err := sys.PackageInstalled(ctx, run, r.Package); err != nil {
|
||||
return out, err
|
||||
} else if still {
|
||||
return out, fmt.Errorf("%s was removed without error and the package database still has it", r.Package)
|
||||
}
|
||||
out.Action = "removed"
|
||||
out.Detail = "declared absent; its configuration is left where the package manager leaves it"
|
||||
return out, nil
|
||||
}
|
||||
if installed {
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "already installed"
|
||||
@@ -2243,7 +2216,6 @@ func declaredDigest(r declaration.Resource) string {
|
||||
// what was actually done — a unit already as it was found is forgotten, not "restored".
|
||||
func removeService(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
made bool) (string, string, error) {
|
||||
run = managerFor(a.Scope, a.User, run)
|
||||
if a.Stateless {
|
||||
// Declared with no state (novox/hq ADR 0117): its lifecycle was never the mesh's, and the
|
||||
// declaration that said so is the operator's word to hold to, a file of the mesh's or not.
|
||||
@@ -2418,23 +2390,3 @@ func moduleOf(identity string) (string, bool) {
|
||||
}
|
||||
return identity[:at], true
|
||||
}
|
||||
|
||||
// managerFor routes a unit's commands to the manager it belongs to (novox/hq ADR 0177). A system
|
||||
// unit's go to the machine's manager as they always did. A user-scoped unit's go to the account's
|
||||
// own: `systemctl --user --machine=<account>@`, which reaches that manager from the host's own
|
||||
// process without an environment to forge or a user to switch to — and which only answers while
|
||||
// the account's manager runs (a login, or lingering enabled for the account). Done on the runner
|
||||
// rather than in each system: every system's reading of a unit already goes through `systemctl`,
|
||||
// so this is one place instead of one per system and one per method.
|
||||
func managerFor(scope, user string, run Runner) Runner {
|
||||
if scope != declaration.ScopeUser || user == "" {
|
||||
return run
|
||||
}
|
||||
return func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
if name != "systemctl" {
|
||||
return run(ctx, name, args...)
|
||||
}
|
||||
scoped := append([]string{"--user", "--machine=" + user + "@"}, args...)
|
||||
return run(ctx, name, scoped...)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,42 +173,3 @@ func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
|
||||
t.Fatal("a capability is not part of the container's spec")
|
||||
}
|
||||
}
|
||||
|
||||
// A package may be declared absent (novox/hq ADR 0175): removed when it is installed, read back,
|
||||
// left alone when it is not.
|
||||
func TestAPackageDeclaredAbsentIsRemovedWhenPresentAndLeftWhenNot(t *testing.T) {
|
||||
installed := true
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
if name != "pacman" {
|
||||
return "", nil
|
||||
}
|
||||
switch args[0] {
|
||||
case "-Q":
|
||||
if args[1] == "pacman" || installed {
|
||||
return args[1] + " 1.0\n", nil
|
||||
}
|
||||
return "", errors.New("package not found")
|
||||
case "-R":
|
||||
installed = false
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
d := parseTrusted(t, `{"declaration":1,"resources":[{"id":"front-end","type":"package","package":"ufw","absent":true}]}`)
|
||||
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Outcomes[0].Action != "removed" || !strings.Contains(strings.Join(ran, "\n"), "pacman -R --noconfirm ufw") {
|
||||
t.Fatalf("an installed package declared absent was not removed: %+v\n%v", report.Outcomes[0], ran)
|
||||
}
|
||||
ran = nil
|
||||
report, _, err = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Outcomes[0].Action != "unchanged" || strings.Contains(strings.Join(ran, "\n"), "-R") {
|
||||
t.Fatalf("a package already absent was touched: %+v\n%v", report.Outcomes[0], ran)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// A scheduled step may hold its module's own containers still while it runs (novox/hq ADR 0189,
|
||||
// issue 108).
|
||||
//
|
||||
// What it exists for: the artifact store's collector walks the storage and requires every writer
|
||||
// stopped. A run-once step runs beside containers and a scheduled one is the same container again,
|
||||
// so the mesh had no way to say it — which is why the store it inherited has never collected
|
||||
// anything. The risk the field brings is one shape only: a window that opens and never closes.
|
||||
// Every test here is about that shape.
|
||||
|
||||
// windowRun records the order of stop / run / start, which is the whole of what is being asserted.
|
||||
type windowRun struct {
|
||||
mu sync.Mutex
|
||||
order []string
|
||||
failAt string // the arg[0] that should fail ("run" makes the step fail)
|
||||
wontGo string // a container name that refuses to start again
|
||||
}
|
||||
|
||||
func (w *windowRun) run(_ context.Context, _ string, args ...string) (string, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch args[0] {
|
||||
case "info":
|
||||
return "27.0\n", nil
|
||||
case "stop", "start":
|
||||
w.order = append(w.order, args[0]+" "+args[1])
|
||||
if args[0] == "start" && args[1] == w.wontGo {
|
||||
return "", errors.New("the runtime refused")
|
||||
}
|
||||
case "run":
|
||||
w.order = append(w.order, "run")
|
||||
if w.failAt == "run" {
|
||||
return "", errors.New("the step exited non-zero")
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (w *windowRun) seen() []string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return append([]string{}, w.order...)
|
||||
}
|
||||
|
||||
// aStoreWithACollector is a module in the shape distribution has: a server that must not be
|
||||
// writing, and a nightly step that walks its storage with the server held still.
|
||||
func aStoreWithACollector(t *testing.T) *declaration.Declaration {
|
||||
t.Helper()
|
||||
return parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
|
||||
{"id":"collect","type":"container","name":"mesh-registry-collect","image":"`+pinned+`",
|
||||
"schedule":"30 3 * * *","while-stopped":["store"]}
|
||||
]}`)
|
||||
}
|
||||
|
||||
func fireOnce(t *testing.T, d *declaration.Declaration, w *windowRun) {
|
||||
t.Helper()
|
||||
clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
|
||||
s := NewScheduler(clock, w.run, func(string) {})
|
||||
s.Sync(d, nil)
|
||||
s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
|
||||
s.Wait()
|
||||
}
|
||||
|
||||
func TestAScheduledStepHoldsItsModulesContainerStillAndStartsItAgain(t *testing.T) {
|
||||
w := &windowRun{}
|
||||
fireOnce(t, aStoreWithACollector(t), w)
|
||||
|
||||
got := w.seen()
|
||||
want := []string{"stop mesh-registry", "run", "start mesh-registry"}
|
||||
var kept []string
|
||||
for _, line := range got {
|
||||
if strings.HasPrefix(line, "stop mesh-registry-collect") {
|
||||
// Clearing the step's own exited container by name; not part of the window.
|
||||
continue
|
||||
}
|
||||
kept = append(kept, line)
|
||||
}
|
||||
if len(kept) != len(want) {
|
||||
t.Fatalf("the window was not stop, run, start: %v", got)
|
||||
}
|
||||
for i := range want {
|
||||
if kept[i] != want[i] {
|
||||
t.Fatalf("the window was %v, want %v", kept, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The one that matters: a step that fails must leave the service running.
|
||||
func TestAFailedStepStillClosesTheWindow(t *testing.T) {
|
||||
w := &windowRun{failAt: "run"}
|
||||
fireOnce(t, aStoreWithACollector(t), w)
|
||||
|
||||
var started bool
|
||||
for _, line := range w.seen() {
|
||||
if line == "start mesh-registry" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
if !started {
|
||||
t.Fatalf("the step failed and the container it held still was never started again: %v", w.seen())
|
||||
}
|
||||
}
|
||||
|
||||
// A container that will not come back is said loudly: it is down, and nothing else notices until
|
||||
// the next apply compares it.
|
||||
func TestAContainerThatWillNotStartAgainIsSaidLoudly(t *testing.T) {
|
||||
w := &windowRun{wontGo: "mesh-registry"}
|
||||
var said []string
|
||||
clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
|
||||
s := NewScheduler(clock, w.run, func(line string) { said = append(said, line) })
|
||||
s.Sync(aStoreWithACollector(t), nil)
|
||||
s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
|
||||
s.Wait()
|
||||
|
||||
var loud bool
|
||||
for _, line := range said {
|
||||
if strings.Contains(line, "WILL NOT START AGAIN") && strings.Contains(line, "mesh-registry") {
|
||||
loud = true
|
||||
}
|
||||
}
|
||||
if !loud {
|
||||
t.Fatalf("a service left stopped by a maintenance window was not said loudly: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// Several containers come back in the reverse of the order they were stopped: a module names the
|
||||
// dependant first, and starting it before what it depends on is not bringing it back.
|
||||
func TestTheWindowClosesInTheReverseOfTheOrderItOpened(t *testing.T) {
|
||||
d := parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"web","type":"container","name":"web","image":"`+pinned+`"},
|
||||
{"id":"db","type":"container","name":"db","image":"`+pinned+`"},
|
||||
{"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
|
||||
"schedule":"30 3 * * *","while-stopped":["web","db"]}
|
||||
]}`)
|
||||
w := &windowRun{}
|
||||
fireOnce(t, d, w)
|
||||
|
||||
var stops, starts []string
|
||||
for _, line := range w.seen() {
|
||||
switch {
|
||||
case line == "stop web" || line == "stop db":
|
||||
stops = append(stops, line)
|
||||
case strings.HasPrefix(line, "start "):
|
||||
starts = append(starts, line)
|
||||
}
|
||||
}
|
||||
if len(stops) != 2 || stops[0] != "stop web" || stops[1] != "stop db" {
|
||||
t.Fatalf("stopped in %v, want the order the step named them", stops)
|
||||
}
|
||||
if len(starts) != 2 || starts[0] != "start db" || starts[1] != "start web" {
|
||||
t.Fatalf("started in %v, want the reverse", starts)
|
||||
}
|
||||
}
|
||||
|
||||
// And the refusals, each for what it says rather than that it says something.
|
||||
func TestAMaintenanceWindowIsRefusedWhereItCannotMean(t *testing.T) {
|
||||
for _, c := range []struct{ name, body, says string }{
|
||||
{
|
||||
"a window with no schedule",
|
||||
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","while-stopped":["store"]}`,
|
||||
"needs a schedule",
|
||||
},
|
||||
{
|
||||
"a window naming itself",
|
||||
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["collect"]}`,
|
||||
"this step itself",
|
||||
},
|
||||
{
|
||||
"a window naming something that is not a container here",
|
||||
`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["elsewhere"]}`,
|
||||
"no container by that id",
|
||||
},
|
||||
} {
|
||||
_, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` + c.body + `]}`))
|
||||
if err == nil {
|
||||
t.Errorf("%s was accepted", c.name)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: the refusal does not say %q: %v", c.name, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,16 +76,6 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri
|
||||
if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive {
|
||||
return "", nil
|
||||
}
|
||||
if !firewall.Installed(ctx, run) {
|
||||
// Uninstalled (novox/hq ADR 0175): retired for good, by the module that replaced it. Said
|
||||
// once, and nothing is asked of a command that is not there.
|
||||
if rec.RetiredBy != firewall.RetiredRemoved {
|
||||
rec.RetiredBy = firewall.RetiredRemoved
|
||||
log(" the found firewall (ufw) is no longer installed; the mesh's filter is what filters this machine")
|
||||
return "removed: ufw is no longer installed; the mesh's filter is what filters this machine", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
active := firewall.Active(ctx, run)
|
||||
if !active && !(rec.Forward != nil && !rec.DisabledByMesh) {
|
||||
// Inactive, and either the mesh's doing already or nobody's recorded here: said as found,
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
@@ -523,33 +522,3 @@ func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
|
||||
t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall)
|
||||
}
|
||||
}
|
||||
|
||||
// A front end that is no longer installed is recorded as removed, said once, and asked nothing of
|
||||
// (novox/hq ADR 0175).
|
||||
func TestAnUninstalledFrontEndIsRetiredForGood(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
u := &ufwMachine{installed: false, ruleset: "table inet mesh\n"}
|
||||
known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true,
|
||||
RetiredBy: "mesh", FoundAt: time.Now()}}
|
||||
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
|
||||
report, state, err := applyWith(t, converged, known, u.run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.Firewall.RetiredBy != "removed" || !strings.Contains(report.Firewall, "no longer installed") {
|
||||
t.Fatalf("record %+v, said %q", state.Firewall, report.Firewall)
|
||||
}
|
||||
u.asked = nil
|
||||
report, _, err = applyWith(t, converged, state, u.run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Firewall != "" {
|
||||
t.Errorf("said again: %q", report.Firewall)
|
||||
}
|
||||
for _, a := range u.asked {
|
||||
if strings.HasPrefix(a, "ufw") && a != "ufw status" {
|
||||
t.Errorf("asked something of a front end that is not there: %v", u.asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,29 @@ type scheduledJob struct {
|
||||
container *declaration.Container
|
||||
next time.Time // the next minute at which it is due
|
||||
running bool // a run is in flight — the next due run is skipped rather than stacked
|
||||
// hold is the runtime names of the containers held still for the duration of a run, in the
|
||||
// order the step named them (novox/hq ADR 0189).
|
||||
hold []string
|
||||
}
|
||||
|
||||
// heldStillFor is the runtime names of the containers a step holds still, resolved from ids.
|
||||
func heldStillFor(step *declaration.Container, d *declaration.Declaration) []string {
|
||||
if len(step.WhileStopped) == 0 {
|
||||
return nil
|
||||
}
|
||||
byID := map[string]string{}
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok {
|
||||
byID[c.Identity()] = c.Name
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(step.WhileStopped))
|
||||
for _, id := range step.WhileStopped {
|
||||
if name := byID[id]; name != "" {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewScheduler builds a scheduler. A nil clock is the system clock; a nil log says nothing.
|
||||
@@ -123,15 +146,20 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
|
||||
// on its cadence rather than staying running to be restarted — so its identity cannot
|
||||
// depend on another resource's content and there is nothing to pass.
|
||||
spec := containerSpec(c, inputs{})
|
||||
// The runtime stops containers by name; the declaration names them by id. Resolved here,
|
||||
// against the declaration this job was armed from, so a fire never has to look anything up
|
||||
// (novox/hq ADR 0189). The parser has already refused an id that is not a container here.
|
||||
hold := heldStillFor(c, d)
|
||||
if existing := s.jobs[c.Identity()]; existing != nil && existing.spec == spec {
|
||||
// Unchanged: keep where it is in its cadence, refresh the declaration pointer only.
|
||||
existing.container = c
|
||||
existing.hold = hold
|
||||
continue
|
||||
}
|
||||
// New or changed: arm it for the next due minute after now.
|
||||
next, _ := cron.Next(s.clock.Now())
|
||||
s.jobs[c.Identity()] = &scheduledJob{
|
||||
id: c.Identity(), spec: spec, cron: cron, container: c, next: next,
|
||||
id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +230,15 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) {
|
||||
return
|
||||
}
|
||||
|
||||
// **The window opens here and closes in the defer, whatever happens** (novox/hq ADR 0189).
|
||||
// Deferred before the first stop so a panic, a failing step or a step that runs long all end
|
||||
// the same way: the service running. The one real risk of this field is a window that never
|
||||
// closes, and the only defence against it is that closing is not conditional on anything.
|
||||
if len(j.hold) > 0 {
|
||||
defer s.letRun(ctx, cri, j)
|
||||
s.holdStill(ctx, cri, j)
|
||||
}
|
||||
|
||||
// A container by this name left exited by the previous run would collide with --name. Removing
|
||||
// one that is not there is the state we want, so its error is ignored — the same as run-once.
|
||||
_, _ = s.run(ctx, cri, "rm", "-f", j.container.Name)
|
||||
@@ -260,3 +297,51 @@ func (s *Scheduler) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// holdStill stops the containers this step runs instead of, in the order it named them.
|
||||
//
|
||||
// A stop that fails is said and not fatal. The step runs anyway: for the case this exists for —
|
||||
// a collector walking storage nothing must be writing to — a writer that would not stop is worth
|
||||
// knowing about, and refusing to run would mean the work never happens and the log says nothing
|
||||
// new each night. What must not be skipped is the restart, and it is not: it is deferred.
|
||||
func (s *Scheduler) holdStill(ctx context.Context, cri string, j *scheduledJob) {
|
||||
for _, name := range j.hold {
|
||||
if _, err := s.run(ctx, cri, "stop", name); err != nil {
|
||||
s.log(fmt.Sprintf("scheduled step %s: could not stop %s for the run: %v", j.id, name, err))
|
||||
continue
|
||||
}
|
||||
s.log(fmt.Sprintf("scheduled step %s: %s held still for the run", j.id, name))
|
||||
}
|
||||
}
|
||||
|
||||
// letRun starts them again, in the reverse of the order they were stopped, and says so loudly if
|
||||
// one does not come back.
|
||||
//
|
||||
// **Reverse order**, because stopping walks a dependency the other way: a module that holds two
|
||||
// containers still names the one that depends on the other first, and bringing them back the same
|
||||
// way would start a dependant before what it depends on.
|
||||
//
|
||||
// Given its own context, because this runs in a defer and the one the run used may already be
|
||||
// cancelled — a host shutting down mid-window would otherwise leave the service stopped, which is
|
||||
// precisely the outcome this field must never have.
|
||||
func (s *Scheduler) letRun(_ context.Context, cri string, j *scheduledJob) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), closingWindow)
|
||||
defer cancel()
|
||||
for i := len(j.hold) - 1; i >= 0; i-- {
|
||||
name := j.hold[i]
|
||||
if _, err := s.run(ctx, cri, "start", name); err != nil {
|
||||
// Said as loudly as this host says anything: a service the mesh stopped for a
|
||||
// maintenance window and could not start again is down, and nothing else will notice
|
||||
// until the next apply compares it.
|
||||
s.log(fmt.Sprintf(
|
||||
"scheduled step %s: %s was held still for the run and WILL NOT START AGAIN: %v",
|
||||
j.id, name, err))
|
||||
continue
|
||||
}
|
||||
s.log(fmt.Sprintf("scheduled step %s: %s running again", j.id, name))
|
||||
}
|
||||
}
|
||||
|
||||
// closingWindow is how long the host will spend putting back what it stopped. Generous: this is
|
||||
// the half that must not be given up on.
|
||||
const closingWindow = 5 * time.Minute
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0177: a unit in the operator account's own service manager is applied
|
||||
// through that manager — `systemctl --user --machine=<account>@` — and never as a system unit of
|
||||
// the same name; its record remembers the scope so removal goes the same way.
|
||||
|
||||
// accountManager is a user's service manager whose one unit starts when asked, recording the
|
||||
// commands and refusing any that reach it outside the account's scope.
|
||||
func accountManager(account string, commands *[]string) Runner {
|
||||
active, enabled := false, false
|
||||
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
*commands = append(*commands, line)
|
||||
if name == "systemctl" && !strings.HasPrefix(line, "systemctl --user --machine="+account+"@ ") {
|
||||
return "", fmt.Errorf("a system-scope command reached the account's manager: %s", line)
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(line, " start "):
|
||||
active = true
|
||||
return "", nil
|
||||
case strings.Contains(line, " stop "):
|
||||
active = false
|
||||
return "", nil
|
||||
case strings.Contains(line, " enable "):
|
||||
enabled = true
|
||||
return "", nil
|
||||
case strings.Contains(line, " disable "):
|
||||
enabled = false
|
||||
return "", nil
|
||||
case strings.Contains(line, "is-enabled"):
|
||||
if enabled {
|
||||
return "enabled", nil
|
||||
}
|
||||
return "disabled", nil
|
||||
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
|
||||
if active {
|
||||
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
|
||||
}
|
||||
return "LoadState=loaded\nActiveState=inactive\nSubState=dead", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestAUserScopedUnitIsAppliedThroughTheAccountsManager(t *testing.T) {
|
||||
var commands []string
|
||||
decl := `{"declaration":1,"resources":[
|
||||
{"id":"i3.watcher","type":"service","unit":"i3-reload-watcher.service","state":"running","boot":"enabled","scope":"user","user":"ops"}
|
||||
]}`
|
||||
report, known, err := Apply(context.Background(), archHost(t), parse(t, decl),
|
||||
store.State{}, store.OriginDeclared, accountManager("ops", &commands), nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("apply: %v\n%s", err, strings.Join(commands, "\n"))
|
||||
}
|
||||
if !report.Changed() {
|
||||
t.Fatal("a unit that was stopped and is now running changed nothing")
|
||||
}
|
||||
var started, enabled bool
|
||||
for _, c := range commands {
|
||||
if c == "systemctl --user --machine=ops@ start i3-reload-watcher.service" {
|
||||
started = true
|
||||
}
|
||||
if c == "systemctl --user --machine=ops@ enable i3-reload-watcher.service" {
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
if !started || !enabled {
|
||||
t.Fatalf("the unit was not started and enabled in the account's manager:\n%s", strings.Join(commands, "\n"))
|
||||
}
|
||||
recorded, ok := known.At("service", "i3-reload-watcher.service")
|
||||
if !ok || recorded.Scope != "user" || recorded.User != "ops" {
|
||||
t.Fatalf("the record does not say whose manager the unit is in: %+v", recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASystemUnitIsUntouchedByTheScope(t *testing.T) {
|
||||
var commands []string
|
||||
decl := `{"declaration":1,"resources":[
|
||||
{"id":"x.daemon","type":"service","unit":"sshd.service","state":"running","boot":"enabled"}
|
||||
]}`
|
||||
if _, _, err := Apply(context.Background(), archHost(t), parse(t, decl),
|
||||
store.State{}, store.OriginDeclared, unitIn(true, &commands), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range commands {
|
||||
if strings.Contains(c, "--user") || strings.Contains(c, "--machine") {
|
||||
t.Fatalf("a system unit was addressed to an account's manager: %s", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -684,16 +684,6 @@ type Service struct {
|
||||
// declaration that reports success and stops being true at the next power cut.
|
||||
Boot string `json:"boot,omitempty"`
|
||||
|
||||
// Scope is whose service manager the unit belongs to: "system" (absent means system), or
|
||||
// "user" — the operator account's own manager (novox/hq ADR 0177). A workstation's per-user
|
||||
// daemons — a window manager's reload watcher, an audio mask, a memory guard — are units in
|
||||
// that manager, and until this they had no form the mesh could send. A user-scoped unit names
|
||||
// its User; the host talks to that account's manager and never starts a system unit by the
|
||||
// same name.
|
||||
Scope string `json:"scope,omitempty"`
|
||||
// User is the account whose manager a user-scoped unit lives in. Required with scope "user",
|
||||
// refused otherwise; a module names it ${machine:account} and never the person.
|
||||
User string `json:"user,omitempty"`
|
||||
// RestartOn names resources whose change means this service must be restarted.
|
||||
//
|
||||
// Because a running service does not re-read its configuration. Replace the file, find the
|
||||
@@ -739,33 +729,11 @@ func (s *Service) Identity() string { return s.ID }
|
||||
func (s *Service) Kind() Type { return TypeService }
|
||||
func (s *Service) Target() string { return s.Unit }
|
||||
|
||||
// ScopeSystem and ScopeUser are the two managers a unit may belong to (novox/hq ADR 0177).
|
||||
const (
|
||||
ScopeSystem = "system"
|
||||
ScopeUser = "user"
|
||||
)
|
||||
|
||||
// UserScoped is whether this unit lives in an account's own service manager.
|
||||
func (s *Service) UserScoped() bool { return s.Scope == ScopeUser }
|
||||
|
||||
func (s *Service) validate(where string, _ bool) []string {
|
||||
var problems []string
|
||||
if s.Unit == "" {
|
||||
problems = append(problems, where+": a service needs a unit")
|
||||
}
|
||||
switch s.Scope {
|
||||
case "", ScopeSystem:
|
||||
if s.User != "" {
|
||||
problems = append(problems, where+": a system unit names no user; only a user-scoped unit does")
|
||||
}
|
||||
case ScopeUser:
|
||||
if s.User == "" {
|
||||
problems = append(problems, where+": a user-scoped unit names the account whose manager it lives in")
|
||||
}
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: scope %q; a unit is in the \"system\" manager or the operator account's \"user\" one", where, s.Scope))
|
||||
}
|
||||
switch {
|
||||
case s.State == "running" || s.State == "stopped":
|
||||
case s.State != "":
|
||||
@@ -902,12 +870,6 @@ type Package struct {
|
||||
ID string `json:"id"`
|
||||
Type Type `json:"type"`
|
||||
Package string `json:"package"`
|
||||
// Absent declares that the package is NOT installed (novox/hq ADR 0175): the host removes it
|
||||
// when it is, and leaves a machine that never had it alone. For the one case a module replaces
|
||||
// software the machine was found with and the operator has decided it does not come back — the
|
||||
// firewall front end a converged machine's filter module retired. Nothing to undo when the
|
||||
// declaration drops it: the host does not install what a declaration stopped saying is absent.
|
||||
Absent bool `json:"absent,omitempty"`
|
||||
}
|
||||
|
||||
func (p *Package) Identity() string { return p.ID }
|
||||
@@ -1035,6 +997,24 @@ type Container struct {
|
||||
// rather than stacked. It is exclusive with RunOnce and with restart-on: a container runs once
|
||||
// and gates, runs on a cadence, or stays up — never two of these.
|
||||
Schedule string `json:"schedule,omitempty"`
|
||||
|
||||
// WhileStopped names resources of the same module — containers — that must be held still for
|
||||
// the duration of this step's run (novox/hq ADR 0189). The host stops each before the run and
|
||||
// starts each again after it, **whatever the step did**: a step that failed must leave the
|
||||
// service running, because the one real risk of this field is a window that never closes.
|
||||
//
|
||||
// **For the work a service cannot have done underneath it.** The artifact store's collector
|
||||
// walks the storage and requires every writer stopped; a run-once step runs beside containers
|
||||
// and a scheduled one is the same container again, so until this there was no way for a module
|
||||
// to say it. The predecessor said it with a shell script, which is how the mesh inherited a
|
||||
// store that has never collected anything.
|
||||
//
|
||||
// **Its own module's containers, and only on a schedule.** A module that could quiesce a
|
||||
// neighbour could stop the mesh. And at apply time the host already has a window — the
|
||||
// declaration is applied in order and a run-once step gates what follows — so a one-time
|
||||
// offline job says *before*, not *instead of*; a recurring window is the case order cannot
|
||||
// express, and the only one this serves.
|
||||
WhileStopped []string `json:"while-stopped,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Container) Identity() string { return c.ID }
|
||||
@@ -1066,6 +1046,20 @@ func (c *Container) validate(where string, _ bool) []string {
|
||||
problems = append(problems, where+": "+err.Error())
|
||||
}
|
||||
}
|
||||
// A maintenance window belongs to a recurring step (novox/hq ADR 0189). Refused on anything
|
||||
// else here, where the field is; that it names containers of the same module, and not itself,
|
||||
// is judged against the whole declaration (see whileStoppedNames).
|
||||
if len(c.WhileStopped) > 0 && c.Schedule == "" {
|
||||
problems = append(problems, where+": while-stopped needs a schedule; at apply the host "+
|
||||
"already has a window — the declaration is applied in order and a run-once step gates "+
|
||||
"what follows — so a one-time offline job is declared before what it works on")
|
||||
}
|
||||
for _, id := range c.WhileStopped {
|
||||
if id == c.ID {
|
||||
problems = append(problems, where+": while-stopped names "+strconv.Quote(id)+
|
||||
", which is this step itself")
|
||||
}
|
||||
}
|
||||
// The runtime's flags take addresses, and a name here would be handed to it verbatim and
|
||||
// refused at create — after the old container was already removed. Refused on arrival instead.
|
||||
for _, d := range c.Dns {
|
||||
@@ -1476,6 +1470,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
problems = append(problems, resource.validate(where, allowActions)...)
|
||||
d.Resources = append(d.Resources, resource)
|
||||
}
|
||||
problems = append(problems, checkWhileStopped(d.Resources)...)
|
||||
problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
|
||||
if env.Adoption == nil {
|
||||
for _, r := range d.Resources {
|
||||
@@ -1504,6 +1499,41 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// checkWhileStopped judges a maintenance window against the whole declaration (novox/hq ADR 0189).
|
||||
//
|
||||
// A step may hold still only a container that is **here** — in this same declaration, which is to
|
||||
// say on this machine and placed by the mesh. That is what makes it the module's own: a node's
|
||||
// declaration carries one module's resources beside another's, so the id must also be a container
|
||||
// and not a file or a directory, which there would be nothing to stop.
|
||||
//
|
||||
// Refused on arrival rather than discovered at the first fire. A window that names something the
|
||||
// host cannot stop is a window that opens at 03:00 and reports nothing until somebody reads a log.
|
||||
func checkWhileStopped(resources []Resource) []string {
|
||||
containers := map[string]bool{}
|
||||
for _, r := range resources {
|
||||
if r.Kind() == TypeContainer {
|
||||
containers[r.Identity()] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, r := range resources {
|
||||
c, ok := r.(*Container)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, id := range c.WhileStopped {
|
||||
if containers[id] {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"resource %q: while-stopped names %q, and this declaration has no container by "+
|
||||
"that id. A step may hold still only a container placed on this machine "+
|
||||
"beside it", c.ID, id))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func strictDecode(raw []byte, into any) error {
|
||||
// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
|
||||
// field the host does not know is a thing the control plane believes it asked for.
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0177: a unit is in the system manager or an account's own; a user-scoped one names
|
||||
// the account, a system one may not, and any other word is refused.
|
||||
func TestAUserScopedUnitNamesItsAccountAndASystemOneMayNot(t *testing.T) {
|
||||
cases := []struct{ scope, user, wants string }{
|
||||
{"user", "ops", ""},
|
||||
{"", "", ""},
|
||||
{"system", "", ""},
|
||||
{"user", "", "names the account"},
|
||||
{"", "ops", "names no user"},
|
||||
{"session", "ops", "scope \"session\""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s := &Service{ID: "m.u", Type: TypeService, Unit: "u.service", State: "running", Scope: c.scope, User: c.user}
|
||||
problems := s.validate("m.u", false)
|
||||
got := strings.Join(problems, "; ")
|
||||
if c.wants == "" && len(problems) != 0 {
|
||||
t.Fatalf("scope %q user %q refused: %s", c.scope, c.user, got)
|
||||
}
|
||||
if c.wants != "" && !strings.Contains(got, c.wants) {
|
||||
t.Fatalf("scope %q user %q: wanted a refusal saying %q, got %q", c.scope, c.user, c.wants, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -319,17 +319,8 @@ func Active(ctx context.Context, run Runner) bool {
|
||||
return err == nil && statusActive(out)
|
||||
}
|
||||
|
||||
// Installed says whether ufw is on this machine at all: a command that is not there is a front end
|
||||
// that was uninstalled (novox/hq ADR 0175), not one that is silent.
|
||||
func Installed(ctx context.Context, run Runner) bool {
|
||||
_, err := run(ctx, "ufw", "status")
|
||||
return !missing(err)
|
||||
}
|
||||
|
||||
// Retirements of a found firewall, as the host records them.
|
||||
const (
|
||||
RetiredByMesh = "mesh"
|
||||
RetiredFoundSo = "found-inactive"
|
||||
// RetiredRemoved is a front end uninstalled by the module that replaced it (ADR 0175).
|
||||
RetiredRemoved = "removed"
|
||||
)
|
||||
|
||||
@@ -82,10 +82,6 @@ type Applied struct {
|
||||
// 0117) — kept here because removal happens once the declaration that said so is gone, and a
|
||||
// service removed as if it had a state is stopped: the machine's network manager, for one.
|
||||
Stateless bool `json:"stateless,omitempty"`
|
||||
// Scope and User are, for a service in an account's own manager (novox/hq ADR 0177), which
|
||||
// manager — so removal gives the unit back through the same one it was applied through.
|
||||
Scope string `json:"scope,omitempty"`
|
||||
User string `json:"user,omitempty"`
|
||||
|
||||
// Found is, for a service, the state its unit was in when this host first applied it — before
|
||||
// the mesh started, stopped, enabled or disabled anything. Removal gives that back and nothing
|
||||
|
||||
@@ -46,11 +46,6 @@ func (a alpine) PackageInstalled(ctx context.Context, run Runner, name string) (
|
||||
return strings.TrimSpace(out) != "", nil
|
||||
}
|
||||
|
||||
func (alpine) RemovePackage(ctx context.Context, run Runner, name string) error {
|
||||
_, err := run(ctx, "apk", "del", name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (alpine) InstallPackage(ctx context.Context, run Runner, name string) error {
|
||||
_, err := run(ctx, "apk", "add", "--no-cache", name)
|
||||
return err
|
||||
|
||||
@@ -65,10 +65,6 @@ func (a android) InstallPackage(context.Context, Runner, string) error {
|
||||
return fmt.Errorf("%w: package", ErrUnsupported)
|
||||
}
|
||||
|
||||
func (a android) RemovePackage(context.Context, Runner, string) error {
|
||||
return fmt.Errorf("%w: package", ErrUnsupported)
|
||||
}
|
||||
|
||||
func (a android) ServiceState(context.Context, Runner, string) (string, error) {
|
||||
return "", fmt.Errorf("%w: service (init is not reachable without root)", ErrUnsupported)
|
||||
}
|
||||
|
||||
@@ -39,14 +39,6 @@ func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bo
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// RemovePackage removes one package and nothing it depends on: `-R`, not `-Rs`, because what else
|
||||
// relied on a dependency is not this declaration's to know. pacman keeps a configuration file the
|
||||
// operator changed as `.pacsave`, which is what "never flushed" comes to once the front end is gone.
|
||||
func (arch) RemovePackage(ctx context.Context, run Runner, name string) error {
|
||||
_, err := run(ctx, "pacman", "-R", "--noconfirm", name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
|
||||
out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name)
|
||||
if err == nil {
|
||||
|
||||
@@ -51,9 +51,6 @@ type System interface {
|
||||
|
||||
PackageInstalled(ctx context.Context, run Runner, name string) (bool, error)
|
||||
InstallPackage(ctx context.Context, run Runner, name string) error
|
||||
// RemovePackage uninstalls one package, leaving its dependencies and anything the operator
|
||||
// changed in its configuration where the package manager leaves them (novox/hq ADR 0175).
|
||||
RemovePackage(ctx context.Context, run Runner, name string) error
|
||||
|
||||
// ServiceState is "running" or "stopped". A unit that does not exist is an error, never
|
||||
// "stopped" — reporting absence as satisfaction is the fault this host exists to prevent.
|
||||
|
||||
Reference in New Issue
Block a user