Files
mesh-host/internal/apply/hold_test.go
T
jschoubben fc593b9dfd Stop nothing the mesh cannot replace, give the tunnel back on failure, and take it over after enrolment
Review of the ADR 0105 build (hq ADR 0105). The takeover stopped the found
unit and then found out whether the mesh's interface would do; a start that
failed left the machine with no tunnel at all.

Now nothing is stopped until the declared interface listens on the found port
at the found address and the key file it names holds the found key — the
refusal names the remedy — and a mesh interface that fails to start after the
takeover has the found unit started again, with the account saying so. The
account has three states (not taken, taken, down) and is given on every
takeover, failure included. An interface raised by hand is looked at again
for a moment and then refused naming `wg-quick down`. A found unit started
again by hand beside the mesh's is said, not stopped: on the hub it cannot
hold the port, and on a spoke two interfaces with one key would fight.

`mesh-host overlay take --tunnel <iface>` is the path for a node that
enrolled before the mesh knew to take a tunnel over: the found key becomes its
overlay key — identity, sealing and serving keys untouched, so nothing sealed
to the node is remade — and the mesh is told with a rekey signed by the
identity key, over the key left, the key taken and the tunnel. Told first,
written second, so a run again puts right whichever half did not happen.
2026-09-24 00:02:08 +02:00

1053 lines
43 KiB
Go

package apply
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0100: on an adopted node, what is found is kept until its module is taken.
// machine is a fake container runtime holding containers by name: id, running, and the host's spec
// label when a host made it. Every command it is asked is written down.
type machine struct {
containers map[string]*fakeContainer
asked []string
// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
wgUp string
// units are service units by name, as systemd would report them; volumes are the runtime's
// named volumes.
units map[string]*fakeUnit
volumes map[string]bool
users map[string]bool
}
type fakeUnit struct {
active, enabled string
// wontStart is a unit that accepts `start` and stays inactive — one that starts and dies.
wontStart bool
// fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an
// administrator installs one.
fragment string
}
// systemctl answers as systemd does for the units the machine has, and "not-found" for any other.
func (m *machine) systemctl(args []string) (string, error) {
unit := args[len(args)-1]
if args[0] == "show" {
unit = args[1]
}
u, ok := m.units[unit]
switch args[0] {
case "show":
if !ok {
if len(args) > 2 && strings.Contains(args[2], "FragmentPath") {
return "FragmentPath=\n", nil
}
return "LoadState=not-found\nActiveState=inactive\nType=simple\n", nil
}
if len(args) > 2 && strings.Contains(args[2], "FragmentPath") {
from := u.fragment
if from == "" {
from = "/etc/systemd/system/" + unit
}
return "FragmentPath=" + from + "\n", nil
}
return "LoadState=loaded\nActiveState=" + u.active + "\nType=simple\nRemainAfterExit=no\n", nil
case "is-enabled":
if !ok {
return "", errors.New("not found")
}
return u.enabled + "\n", nil
case "start":
if !u.wontStart {
u.active = "active"
}
case "stop":
u.active = "inactive"
case "enable":
u.enabled = "enabled"
case "disable":
u.enabled = "disabled"
}
return "", nil
}
func (m *machine) did(prefix string) bool {
for _, a := range m.asked {
if strings.HasPrefix(a, prefix) {
return true
}
}
return false
}
type fakeContainer struct {
id string
running bool
spec string
}
func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
m.asked = append(m.asked, name+" "+strings.Join(args, " "))
if name == "systemctl" {
return m.systemctl(args)
}
if name == "wg" {
return m.wgUp, nil
}
if name == "getent" {
if m.users[args[len(args)-1]] {
return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil
}
return "", errors.New("exit status 2")
}
if name != "docker" {
return "", nil
}
switch args[0] {
case "volume":
if m.volumes[args[len(args)-1]] {
return "[]\n", nil
}
return "", errors.New("no such volume")
case "info":
return "27.0\n", nil
case "inspect":
c, ok := m.containers[args[len(args)-1]]
if !ok {
return "", errors.New("no such container")
}
running := "false"
if c.running {
running = "true"
}
if strings.HasPrefix(args[2], "{{.Id}}") {
return c.id + "\t" + running + "\t" + c.spec + "\n", nil
}
return running + "\t" + c.spec + "\n", nil
case "rm":
delete(m.containers, args[len(args)-1])
return "", nil
case "run":
var name, spec string
for i, a := range args {
if a == "--name" {
name = args[i+1]
}
if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
spec = strings.TrimPrefix(args[i+1], specLabel+"=")
}
}
m.containers[name] = &fakeContainer{id: "made-by-host", running: true, spec: spec}
return "made-by-host\n", nil
}
return "", nil
}
func (m *machine) removed(name string) bool {
for _, a := range m.asked {
if strings.HasPrefix(a, "docker rm") && strings.HasSuffix(a, " "+name) {
return true
}
}
return false
}
func adopted(t *testing.T, adoption, resources string) *declaration.Declaration {
t.Helper()
return parse(t, `{"declaration":1,"adoption":`+adoption+`,"resources":[`+resources+`]}`)
}
const untakenWeb = `{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}`
const takenWeb = `{"taken":["hello-web"]}`
func webResources(page string) string {
return `{"id":"hello-web.page","type":"file","path":"` + page + `","content":"the mesh's page\n"},
{"id":"hello-web.server","type":"container","name":"hello-web","image":"` + pinned + `"}`
}
func applyAdopted(t *testing.T, d *declaration.Declaration, known store.State, m *machine, keepDir string) (Report, store.State) {
t.Helper()
report, state, err := ApplyKeeping(context.Background(), archHost(t), d, known,
store.OriginDeclared, m.run, nil, nil, KeepIn(keepDir))
if err != nil {
t.Fatalf("apply failed: %v", err)
}
return report, state
}
func outcomeOf(r Report, id string) Outcome {
for _, o := range r.Outcomes {
if o.ID == id {
return o
}
}
return Outcome{}
}
func predecessor(t *testing.T) (dir, page string, m *machine) {
t.Helper()
dir = t.TempDir()
page = filepath.Join(dir, "index.html")
if err := os.WriteFile(page, []byte("the predecessor's page\n"), 0o640); err != nil {
t.Fatal(err)
}
return dir, page, &machine{containers: map[string]*fakeContainer{
"hello-web": {id: "predecessor-id", running: true},
}}
}
func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
dir, page, m := predecessor(t)
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
got, _ := os.ReadFile(page)
if string(got) != "the predecessor's page\n" {
t.Fatalf("a found file was changed: %q", got)
}
info, _ := os.Stat(page)
if info.Mode().Perm() != 0o640 {
t.Errorf("a found file's mode was changed to %o", info.Mode().Perm())
}
if o := outcomeOf(report, "hello-web.page"); o.Action != "held" ||
!strings.Contains(o.Detail, "kept until hello-web is taken") {
t.Errorf("the found file was not reported held: %+v", o)
}
h, ok := state.HeldAt("hello-web.page")
if !ok || h.Module != "hello-web" || h.Mode != "0640" {
t.Fatalf("the hold was not recorded: %+v", h)
}
kept, err := os.ReadFile(h.Kept)
if err != nil || string(kept) != "the predecessor's page\n" {
t.Fatalf("the original was not kept: %q %v", kept, err)
}
if info, _ := os.Stat(h.Kept); info.Mode().Perm() != 0o600 {
t.Errorf("the kept original is mode %o", info.Mode().Perm())
}
if _, recorded := state.Find("hello-web.page"); recorded {
t.Error("a held file was recorded as applied, so it would be removed as an orphan")
}
if report.Changed() {
t.Errorf("holding was reported as changing the machine: %+v", report.Outcomes)
}
}
func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
dir, page, m := predecessor(t)
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
if m.removed("hello-web") {
t.Fatal("a found container was removed")
}
for _, a := range m.asked {
if strings.HasPrefix(a, "docker run") {
t.Fatalf("a container was started over a found one: %s", a)
}
}
if outcomeOf(report, "hello-web.server").Action != "held" {
t.Errorf("the found container was not held: %+v", report.Outcomes)
}
if h, _ := state.HeldAt("hello-web.server"); h.Container != "predecessor-id" || !h.Running {
t.Errorf("the container as found was not recorded: %+v", h)
}
}
func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
// Assigning prepares: what the module declares that is not there is made.
dir := t.TempDir()
page := filepath.Join(dir, "index.html")
m := &machine{containers: map[string]*fakeContainer{}}
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.page"); o.Action != "created" {
t.Errorf("an absent file of an untaken module was not created: %+v", o)
}
if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
t.Errorf("an absent container of an untaken module was not created: %+v", o)
}
if len(state.Held) != 0 {
t.Errorf("something was held that was not found: %+v", state.Held)
}
}
func TestAFileThisHostWroteIsNotFound(t *testing.T) {
// Found means present with no record. A record of any origin — carried or declared, this life
// of the node or an earlier one — means this host wrote it.
for _, origin := range []string{store.OriginCarried, store.OriginDeclared} {
dir, page, m := predecessor(t)
known := store.State{Resources: []store.Applied{
{ID: "earlier-name", Type: "file", Target: page, Origin: origin}}}
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), known, m, dir)
if o := outcomeOf(report, "hello-web.page"); o.Action == "held" {
t.Errorf("%s: a file this host has a record of was held: %+v", origin, o)
}
if _, held := state.HeldAt("hello-web.page"); held {
t.Errorf("%s: a recorded file was held", origin)
}
}
}
func TestAContainerAHostMadeIsNotFound(t *testing.T) {
dir, page, m := predecessor(t)
m.containers["hello-web"].spec = "some-spec"
report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.server"); o.Action == "held" {
t.Errorf("a container carrying the host's spec label was held: %+v", o)
}
}
func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
// ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted
// as a module by name. It is the mesh's own and must never read as a predecessor's.
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{"mesh-store": {id: "x", running: true}}}
known := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}
d := adopted(t, `{"taken":[],"untaken":{"postgres":["postgres.server"]}}`,
`{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`"}`)
report, state := applyAdopted(t, d, known, m, dir)
if o := outcomeOf(report, "postgres.server"); o.Action == "held" {
t.Errorf("the carried store was held: %+v", o)
}
if len(state.Held) != 0 {
t.Errorf("the carried store was held: %+v", state.Held)
}
}
func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
h, _ := state.HeldAt("hello-web.page")
report, state := applyAdopted(t, adopted(t, takenWeb, webResources(page)), state, m, dir)
got, _ := os.ReadFile(page)
if string(got) != "the mesh's page\n" {
t.Fatalf("taking the module did not converge the file: %q", got)
}
if !m.removed("hello-web") || m.containers["hello-web"].id != "made-by-host" {
t.Fatal("taking the module did not replace the found container")
}
if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "original kept at "+h.Kept) {
t.Errorf("the cutover does not say where the original is: %+v", o)
}
if len(state.Held) != 0 {
t.Errorf("what was taken is still held: %+v", state.Held)
}
if _, recorded := state.Find("hello-web.page"); !recorded {
t.Error("a taken file was not recorded as applied")
}
kept, err := os.ReadFile(h.Kept)
if err != nil || string(kept) != "the predecessor's page\n" {
t.Errorf("the kept original did not survive the cutover: %q %v", kept, err)
}
}
func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
other := filepath.Join(dir, "other")
_, state = applyAdopted(t, adopted(t, `{"taken":[]}`,
`{"id":"x.other","type":"file","path":"`+other+`","content":"x"}`), state, m, dir)
if got, _ := os.ReadFile(page); string(got) != "the predecessor's page\n" {
t.Fatalf("a held file was touched when its module left: %q", got)
}
if m.removed("hello-web") {
t.Fatal("a held container was removed when its module left")
}
if _, still := state.HeldAt("hello-web.page"); still {
t.Error("a hold outlived its resource leaving the declaration, so the node reports it for ever")
}
if _, recorded := state.Find("hello-web.page"); recorded {
t.Error("a file let go was recorded as the mesh's")
}
}
func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
first, _ := state.HeldAt("hello-web.page")
report, state := applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir)
if o := outcomeOf(report, "hello-web.page"); o.Action != "forgotten" || o.Detail != "no longer declared; left as found" {
t.Errorf("letting a hold go was not reported: %+v", o)
}
if len(state.Held) != 0 {
t.Errorf("holds outlived their resources: %+v", state.Held)
}
// The module comes back: what is there is found again, and the original first kept stays.
if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
t.Fatal(err)
}
report, state = applyAdopted(t, adopted(t, untakenWeb, webResources(page)), state, m, dir)
if o := outcomeOf(report, "hello-web.page"); o.Action != "held" {
t.Fatalf("a returning module's found file was not held again: %+v", o)
}
again, _ := state.HeldAt("hello-web.page")
// The predecessor rewrote it while nothing held it, so that is a second original, kept beside
// the first rather than in place of it (novox/hq ADR 0100).
if kept, _ := os.ReadFile(again.Kept); string(kept) != "the predecessor wrote again\n" {
t.Errorf("what is named as kept is %q", kept)
}
if kept, _ := os.ReadFile(first.Kept); string(kept) != "the predecessor's page\n" {
t.Errorf("the first kept original was lost: %q", kept)
}
if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" {
t.Errorf("the found file was touched: %q", got)
}
}
func TestACarriedApplyLetsNoHoldGo(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
_, state, err := ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried,
m.run, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(state.Held) != 2 {
t.Errorf("a carried apply let holds go: %+v", state.Held)
}
}
func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
dir, page, m := predecessor(t)
d := adopted(t, untakenWeb, webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
t.Fatal(err)
}
report, state := applyAdopted(t, d, state, m, dir)
if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" {
t.Fatalf("a held file was reverted: %q", got)
}
if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() {
t.Errorf("a rewrite was not recorded: %+v", h)
}
if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") {
t.Errorf("a rewrite was not reported: %+v", o)
}
h, _ := state.HeldAt("hello-web.page")
if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" {
t.Errorf("the kept original was overwritten by a later write: %q", kept)
}
}
func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
for _, c := range []struct {
change func(*machine)
want string
}{
{func(m *machine) { m.containers["hello-web"].running = false }, "stopped"},
{func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"},
{func(m *machine) { delete(m.containers, "hello-web") }, "gone"},
} {
dir, page, m := predecessor(t)
d := adopted(t, untakenWeb, webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
c.change(m)
m.asked = nil
_, state = applyAdopted(t, d, state, m, dir)
if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want {
t.Errorf("%s: recorded as %q", c.want, h.Changed)
}
for _, a := range m.asked {
if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") ||
strings.HasPrefix(a, "docker start") {
t.Errorf("%s: the held container was acted on: %s", c.want, a)
}
}
}
}
func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
dir, page, m := predecessor(t)
d := adopted(t, untakenWeb, webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
if err := os.Remove(page); err != nil {
t.Fatal(err)
}
_, state = applyAdopted(t, d, state, m, dir)
if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) {
t.Fatal("a held file that vanished was created before its module was taken")
}
if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" {
t.Errorf("a vanished held file was not reported gone: %+v", h)
}
}
func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
dir, page, m := predecessor(t)
d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
report, state := applyAdopted(t, d, store.State{}, m, dir)
if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" {
t.Errorf("a converged node kept a found file: %q", got)
}
if !m.removed("hello-web") {
t.Error("a converged node kept a found container")
}
if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" {
t.Errorf("a converged node held something: %+v", state.Held)
}
// What it writes over that it has no record of, it keeps first — on any node.
o := outcomeOf(report, "hello-web.page")
kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):]
if got, err := os.ReadFile(kept); err != nil || string(got) != "the predecessor's page\n" {
t.Errorf("the file written over was not kept, or not named: %q (%s) %v", got, o.Detail, err)
}
}
func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) {
dir := t.TempDir()
conf := filepath.Join(dir, "nftables.conf")
_ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644)
decl := func(content string) *declaration.Declaration {
return parse(t, `{"declaration":1,"resources":[{"id":"nftables.config","type":"file","path":"`+conf+
`","content":"`+content+`"}]}`)
}
m := &machine{containers: map[string]*fakeContainer{}}
report, state := applyAdopted(t, decl("table inet mesh {}\\n"), store.State{}, m, dir)
o := outcomeOf(report, "nftables.config")
if !strings.Contains(o.Detail, "had no record of, was kept at ") {
t.Fatalf("writing over an unrecorded file did not keep it: %+v", o)
}
kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):]
if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" {
t.Errorf("the kept original is %q", got)
}
// Now the mesh's: a later change keeps nothing more, and the first original stays.
report, _ = applyAdopted(t, decl("table inet mesh { }\\n"), state, m, dir)
if o := outcomeOf(report, "nftables.config"); o.Action != "updated" || strings.Contains(o.Detail, "kept at") {
t.Errorf("a file the mesh wrote was kept again: %+v", o)
}
if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" {
t.Errorf("the first original was overwritten: %q", got)
}
}
// Defends novox/hq ADR 0103: found covers every kind that can reach what the machine already has.
// untaken is an adoption with hello-web untaken, listing these of its resources.
func untaken(ids ...string) string {
return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}`
}
func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
dir := t.TempDir()
data := filepath.Join(dir, "data")
if err := os.Mkdir(data, 0o700); err != nil {
t.Fatal(err)
}
inside := filepath.Join(data, "PG_VERSION")
if err := os.WriteFile(inside, []byte("16\n"), 0o600); err != nil {
t.Fatal(err)
}
m := &machine{containers: map[string]*fakeContainer{}}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.data"),
`{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), store.State{}, m, dir)
if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 {
t.Errorf("a found directory was re-moded to %o", info.Mode().Perm())
}
if got, _ := os.ReadFile(inside); string(got) != "16\n" {
t.Errorf("what is inside a found directory was touched: %q", got)
}
if o := outcomeOf(report, "hello-web.data"); o.Action != "held" || !strings.Contains(o.Detail, "mode, owner and contents") {
t.Errorf("the found directory was not held: %+v", o)
}
if h, ok := state.HeldAt("hello-web.data"); !ok || h.Mode != "0700" {
t.Errorf("the directory as found was not recorded: %+v", h)
}
if _, recorded := state.Find("hello-web.data"); recorded {
t.Error("a held directory was recorded as applied")
}
}
func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
// A file's parent is made as the file is written; what the mesh made is not found.
dir := t.TempDir()
data := filepath.Join(dir, "data")
m := &machine{containers: map[string]*fakeContainer{}}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.data"),
`{"id":"hello-web.conf","type":"file","path":"`+filepath.Join(data, "conf")+`","content":"x\n"},
{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0750"}`), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.data"); o.Action == "held" {
t.Errorf("a directory the apply itself made was held: %+v", o)
}
if info, _ := os.Stat(data); info.Mode().Perm() != 0o750 {
t.Errorf("the mesh's own directory was not converged: %o", info.Mode().Perm())
}
if len(state.Held) != 0 {
t.Errorf("held: %+v", state.Held)
}
}
func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
dir := t.TempDir()
conf := filepath.Join(dir, "hello.conf")
m := &machine{containers: map[string]*fakeContainer{},
// The predecessor's unit: stopped just now, but it starts at boot, so it is the machine's.
units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}}}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"},
{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled",
"restart-on":["hello-web.conf"]}`), store.State{}, m, dir)
for _, verb := range []string{"systemctl start", "systemctl stop", "systemctl enable", "systemctl disable", "systemctl restart"} {
if m.did(verb) {
t.Errorf("a found service was changed: %s (%v)", verb, m.asked)
}
}
if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "starts at boot") {
t.Errorf("the found service was not held: %+v", o)
}
if h, ok := state.HeldAt("hello-web.unit"); !ok || h.Running {
t.Errorf("the service as found was not recorded: %+v", h)
}
}
func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
// A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service.
dir := t.TempDir()
conf := filepath.Join(dir, "daemon.json")
m := &machine{containers: map[string]*fakeContainer{},
units: map[string]*fakeUnit{"docker.service": {active: "active", enabled: "enabled"}}}
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"{}\n"},
{"id":"hello-web.unit","type":"service","unit":"docker.service","state":"running","boot":"enabled",
"reload-on":["hello-web.conf"]}`), store.State{}, m, dir)
if !m.did("systemctl reload docker.service") {
t.Errorf("a held service was not reloaded for what it re-reads: %v", m.asked)
}
if m.did("systemctl stop") || m.did("systemctl start") {
t.Errorf("a held service was restarted: %v", m.asked)
}
if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "reloaded for hello-web.conf") {
t.Errorf("the reload was not reported on the hold: %+v", o)
}
}
func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
// The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit
// the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and
// the private network never came up. Found is what the machine runs.
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{},
units: map[string]*fakeUnit{"wg-quick@mesh0.service": {active: "inactive", enabled: "disabled",
fragment: "/usr/lib/systemd/system/wg-quick@.service"}}}
report, state := applyAdopted(t, adopted(t, untaken("mesh-wireguard.overlay-up"),
`{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0.service","state":"running","boot":"enabled"}`),
store.State{}, m, dir)
if o := outcomeOf(report, "mesh-wireguard.overlay-up"); o.Action == "held" {
t.Fatalf("a unit nothing runs was held as a predecessor's: %+v", o)
}
if !m.did("systemctl start wg-quick@mesh0.service") || !m.did("systemctl enable wg-quick@mesh0.service") {
t.Errorf("the unit was not started and enabled: %v", m.asked)
}
if len(state.Held) != 0 {
t.Errorf("held: %+v", state.Held)
}
}
func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
// No unit before the apply: nothing of a predecessor's to hold.
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}}
d := adopted(t, untaken("hello-web.unit"), `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running"}`)
_, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
m.run, nil, nil, KeepIn(dir))
if err == nil || !strings.Contains(err.Error(), "does not exist") {
t.Errorf("an absent unit was held rather than applied: %v", err)
}
}
func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
dir := t.TempDir()
data := filepath.Join(dir, "predecessor-data")
if err := os.Mkdir(data, 0o700); err != nil {
t.Fatal(err)
}
for _, c := range []struct {
name, volume string
m *machine
}{
{"a path", data + ":/var/lib/postgresql/data", &machine{containers: map[string]*fakeContainer{}}},
{"a named volume", "predecessor-pgdata:/var/lib/postgresql/data",
&machine{containers: map[string]*fakeContainer{}, volumes: map[string]bool{"predecessor-pgdata": true}}},
} {
report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"),
`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
"volumes":["`+c.volume+`"]}`), store.State{}, c.m, dir)
if c.m.did("docker run") {
t.Errorf("%s: a container mounting found data was created: %v", c.name, c.m.asked)
}
o := outcomeOf(report, "hello-web.server")
if o.Action != "held" || !strings.Contains(o.Detail, "would mount") {
t.Errorf("%s: not held: %+v", c.name, o)
}
if h, ok := state.HeldAt("hello-web.server"); !ok || !strings.Contains(h.Why, "would mount") {
t.Errorf("%s: the hold does not say why: %+v", c.name, h)
}
// Held, it stays held on the next pass, and is still not created.
c.m.asked = nil
_, state = applyAdopted(t, adopted(t, untaken("hello-web.server"),
`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
"volumes":["`+c.volume+`"]}`), state, c.m, dir)
if c.m.did("docker run") || len(state.Held) != 1 {
t.Errorf("%s: a held container was created on the next pass: %v", c.name, c.m.asked)
}
}
}
func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
dir := t.TempDir()
data := filepath.Join(dir, "data")
m := &machine{containers: map[string]*fakeContainer{}}
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data", "hello-web.server"),
`{"id":"hello-web.data","type":"directory","path":"`+data+`"},
{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
"volumes":["`+data+`:/data"]}`), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
t.Errorf("a container mounting only what the mesh made was not created: %+v", o)
}
}
func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
dir, page, m := predecessor(t)
step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `",
"run-once":true,"network":"container:hello-web"}`
report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server", "hello-web.migrate"), webResources(page)+","+step), store.State{}, m, dir)
if m.did("docker run") {
t.Errorf("a step was run inside a held container: %v", m.asked)
}
o := outcomeOf(report, "hello-web.migrate")
if o.Action != "held" || !strings.Contains(o.Detail, "runs in hello-web") {
t.Errorf("the step was not held: %+v", o)
}
if _, ok := state.HeldAt("hello-web.migrate"); !ok {
t.Error("the held step is not reported held")
}
}
func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
// An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the
// host holds one anyway, since what it would run in is the predecessor's.
dir, page, m := predecessor(t)
d := adopted(t, untakenWeb, webResources(page))
d.Resources = append(d.Resources, &declaration.Action{ID: "hello-web.seed", Type: declaration.TypeAction,
In: "hello-web", Command: []string{"seed"}, Verify: []string{"seeded"}})
report, state := applyAdopted(t, d, store.State{}, m, dir)
if m.did("docker exec") {
t.Errorf("an action was run inside a held container: %v", m.asked)
}
if o := outcomeOf(report, "hello-web.seed"); o.Action != "held" || !strings.Contains(o.Detail, "not run until hello-web is taken") {
t.Errorf("the action was not held: %+v", o)
}
if h, ok := state.HeldAt("hello-web.seed"); !ok || h.Module != "hello-web" {
t.Errorf("the held action is not its module's: %+v", h)
}
// Taken: the container is the mesh's, and the action runs in it.
taken := adopted(t, takenWeb, webResources(page))
taken.Resources = append(taken.Resources, d.Resources[len(d.Resources)-1])
report, state = applyAdopted(t, taken, state, m, dir)
if !m.did("docker exec hello-web ") {
t.Errorf("the action did not run once its module was taken: %v", m.asked)
}
if _, still := state.HeldAt("hello-web.seed"); still || len(state.Held) != 0 {
t.Errorf("holds outlived the take: %+v", state.Held)
}
}
const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000"
func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
dir := t.TempDir()
at := filepath.Join(dir, "site")
if err := os.Mkdir(at, 0o750); err != nil {
t.Fatal(err)
}
theirs := filepath.Join(at, "index.html")
_ = os.WriteFile(theirs, []byte("the predecessor's site\n"), 0o640)
m := &machine{containers: map[string]*fakeContainer{}}
// The source is unreachable: fetching it would fail the apply, so a pass means it was not tried.
report, state := applyAdopted(t, adopted(t, untaken("hello-web.site"),
`{"id":"hello-web.site","type":"archive","source":"http://192.0.2.1/site.tar.gz",
"digest":"sha256:`+sixtyFourZeros+`","path":"`+at+`","owner":"root"}`), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.site"); o.Action != "held" || !strings.Contains(o.Detail, "nothing unpacked") {
t.Errorf("an archive over found files was not held: %+v", o)
}
if got, _ := os.ReadFile(theirs); string(got) != "the predecessor's site\n" {
t.Errorf("the found files were touched: %q", got)
}
if _, ok := state.HeldAt("hello-web.site"); !ok {
t.Error("the hold was not recorded")
}
}
func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
dir := t.TempDir()
was := unitDir
unitDir = dir
t.Cleanup(func() { unitDir = was })
unit := filepath.Join(dir, "hello-daemon.service")
_ = os.WriteFile(unit, []byte("[Service]\nExecStart=/opt/predecessor/hello\n"), 0o644)
m := &machine{containers: map[string]*fakeContainer{}}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.daemon"),
`{"id":"hello-web.daemon","type":"process","name":"hello-daemon","source":"http://192.0.2.1/d.tar.gz",
"digest":"sha256:`+sixtyFourZeros+`","run":["hello"]}`), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.daemon"); o.Action != "held" || !strings.Contains(o.Detail, "not written over or restarted") {
t.Errorf("a process whose unit was found was not held: %+v", o)
}
if got, _ := os.ReadFile(unit); string(got) != "[Service]\nExecStart=/opt/predecessor/hello\n" {
t.Errorf("the found unit was written over: %q", got)
}
if m.did("systemctl") {
t.Errorf("the found unit was touched: %v", m.asked)
}
if _, ok := state.HeldAt("hello-web.daemon"); !ok {
t.Error("the hold was not recorded")
}
}
func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}}
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"),
`{"id":"hello-web.user","type":"user","name":"hello","shell":"/bin/zsh","groups":["docker"]}`),
store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.user"); o.Action != "held" || !strings.Contains(o.Detail, "shell and groups") {
t.Errorf("a found user was not held: %+v", o)
}
for _, a := range m.asked {
if strings.HasPrefix(a, "usermod") || strings.HasPrefix(a, "useradd") {
t.Errorf("a found user was changed: %s", a)
}
}
}
func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
// The runtime's socket, the kernel's filesystems and the clock are on every machine; a
// container mounting them shares nothing a predecessor kept (novox/hq ADR 0103).
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{}}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"),
`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
"volumes":["/var/run/docker.sock:/var/run/docker.sock","/etc/localtime:/etc/localtime:ro",
"/proc/cpuinfo:/host/cpuinfo:ro","/dev/null:/data/null"]}`), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
t.Errorf("a container mounting only system paths was not created: %+v", o)
}
if len(state.Held) != 0 {
t.Errorf("held: %+v", state.Held)
}
}
// Defends novox/hq ADR 0100: a runtime that cannot answer is not a machine with nothing there.
// unreachable is a machine whose container runtime answers everything with a daemon that is down.
type unreachable struct{ asked []string }
func (u *unreachable) run(_ context.Context, name string, args ...string) (string, error) {
u.asked = append(u.asked, name+" "+strings.Join(args, " "))
if name == "docker" && args[0] == "info" {
return "27.0\n", nil
}
if name == "docker" {
return "", errors.New("docker exited 1: Cannot connect to the Docker daemon at unix:///var/run/docker.sock")
}
return "", nil
}
func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
dir := t.TempDir()
u := &unreachable{}
d := adopted(t, untaken("hello-web.server"),
`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`"}`)
_, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
u.run, nil, nil, KeepIn(dir))
if err == nil || !strings.Contains(err.Error(), "not safe to make one") {
t.Fatalf("a runtime that could not answer was read as nothing there: %v", err)
}
for _, a := range u.asked {
if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") {
t.Errorf("the mesh acted on a container it could not ask about: %s", a)
}
}
if len(state.Held) != 0 {
t.Errorf("something was held on an answer the machine never gave: %+v", state.Held)
}
}
func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
dir, page, m := predecessor(t)
d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
if _, held := state.HeldAt("hello-web.server"); !held {
t.Fatal("the found container was not held to begin with")
}
u := &unreachable{}
_, state, err := ApplyKeeping(context.Background(), archHost(t), d, state, store.OriginDeclared,
u.run, nil, nil, KeepIn(dir))
if err == nil {
t.Fatal("a runtime that could not answer reported success")
}
if h, held := state.HeldAt("hello-web.server"); !held || h.Changed != "" {
t.Errorf("a hold was let go or called changed on an answer the machine never gave: %+v", h)
}
}
func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
dir := t.TempDir()
run := func(_ context.Context, name string, args ...string) (string, error) {
switch {
case name == "docker" && args[0] == "info":
return "27.0\n", nil
case name == "docker" && args[0] == "volume":
return "", errors.New("docker exited 1: Cannot connect to the Docker daemon")
case name == "docker" && args[0] == "inspect":
return "", errors.New("Error: No such object: hello-web")
case name == "docker":
return "", errors.New("docker run must not happen")
}
return "", nil
}
d := adopted(t, untaken("hello-web.server"),
`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
"volumes":["predecessor-data:/data"]}`)
_, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
run, nil, nil, KeepIn(dir))
if err == nil || !strings.Contains(err.Error(), "could not say whether the volume") {
t.Fatalf("a volume the runtime could not be asked about did not stop the container: %v", err)
}
}
func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
// A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put
// back a service somebody took down on purpose (novox/hq ADR 0103).
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{},
units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled",
fragment: "/etc/systemd/system/hello.service"}}}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.unit"),
`{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`),
store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" {
t.Fatalf("a unit an administrator installed was not held: %+v", o)
}
if m.did("systemctl start") || m.did("systemctl enable") {
t.Errorf("a unit somebody had stopped and disabled was started: %v", m.asked)
}
if _, ok := state.HeldAt("hello-web.unit"); !ok {
t.Error("the hold was not recorded")
}
}
func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
// The predecessor's own service from a package, running: not the mesh's to restart.
dir := t.TempDir()
conf := filepath.Join(dir, "hello.conf")
m := &machine{containers: map[string]*fakeContainer{},
units: map[string]*fakeUnit{"nginx.service": {active: "active", enabled: "enabled",
fragment: "/usr/lib/systemd/system/nginx.service"}}}
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"},
{"id":"hello-web.unit","type":"service","unit":"nginx.service","state":"running","boot":"enabled",
"restart-on":["hello-web.conf"]}`), store.State{}, m, dir)
if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" {
t.Fatalf("a packaged unit the machine runs was not held: %+v", o)
}
if m.did("systemctl stop") || m.did("systemctl restart") {
t.Errorf("the predecessor's service was restarted: %v", m.asked)
}
}
func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
// Held, let go when the module was unassigned, rewritten by the predecessor, found again:
// both originals are kept, and the report names the one it kept (novox/hq ADR 0100).
dir := t.TempDir()
page := filepath.Join(dir, "index.html")
keep := KeepIn(dir)
first, err := keep(page, []byte("the predecessor's page\n"), 0o640)
if err != nil {
t.Fatal(err)
}
same, err := keep(page, []byte("the predecessor's page\n"), 0o640)
if err != nil || same != first {
t.Errorf("the same original was kept twice: %s %s %v", first, same, err)
}
second, err := keep(page, []byte("the predecessor wrote again\n"), 0o640)
if err != nil {
t.Fatal(err)
}
if second == first {
t.Fatal("a second original was kept under the first's name")
}
if got, _ := os.ReadFile(first); string(got) != "the predecessor's page\n" {
t.Errorf("the first original is %q", got)
}
if got, _ := os.ReadFile(second); string(got) != "the predecessor wrote again\n" {
t.Errorf("the second original is %q", got)
}
}
func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)),
store.State{}, m, dir)
first, _ := state.HeldAt("hello-web.page")
// Unassigned, then the predecessor writes again, then assigned once more.
_, state = applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir)
if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
t.Fatal(err)
}
report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)),
state, m, dir)
again, _ := state.HeldAt("hello-web.page")
if again.Kept == first.Kept {
t.Fatalf("the second original was kept under the first's name: %s", again.Kept)
}
if got, _ := os.ReadFile(again.Kept); string(got) != "the predecessor wrote again\n" {
t.Errorf("what the report names as kept is %q", got)
}
if got, _ := os.ReadFile(first.Kept); string(got) != "the predecessor's page\n" {
t.Errorf("the first original was lost: %q", got)
}
if o := outcomeOf(report, "hello-web.page"); o.Action != "held" {
t.Errorf("the file found again was not held: %+v", o)
}
}
func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) {
// An access record says the mesh set permissions on a path it does not own — which is what it
// does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103).
dir := t.TempDir()
data := filepath.Join(dir, "data")
if err := os.Mkdir(data, 0o700); err != nil {
t.Fatal(err)
}
known := store.State{Resources: []store.Applied{
{ID: "hello-web.readable", Type: "access", Target: data, Origin: store.OriginDeclared}}}
m := &machine{containers: map[string]*fakeContainer{}}
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data"),
`{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), known, m, dir)
if o := outcomeOf(report, "hello-web.data"); o.Action != "held" {
t.Errorf("a directory the mesh only has access for was not held: %+v", o)
}
if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 {
t.Errorf("it was re-moded to %o", info.Mode().Perm())
}
}