jschoubben
9254180cab
Review round 1: one quota per unit let three build modules take three shares. The engine now writes meshyield.slice, top-level so its idle weight binds against the operator's session, with the node's share as its CPUQuota; each yielding unit joins it at nice 19. Comments say the IO controls bind only under bfq or io.cost, not under the none scheduler NVMe disks use.
385 lines
16 KiB
Go
385 lines
16 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
func aProcess() *declaration.Process {
|
|
return &declaration.Process{
|
|
ID: "server", Type: declaration.TypeProcess, Name: "greeter",
|
|
Source: "https://store.invalid/greeter/daemon",
|
|
Digest: "sha256:" + strings.Repeat("a", 64),
|
|
Run: []string{"node", "index.js"},
|
|
Env: map[string]string{"MESH_NODE": "anchor", "A_FIRST": "1"},
|
|
}
|
|
}
|
|
|
|
// The unit the mesh writes says what it runs, where, and that it comes back.
|
|
func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
|
unit := unitFor(aProcess())
|
|
for _, want := range []string{
|
|
"ExecStart=node index.js",
|
|
"WorkingDirectory=/var/lib/mesh/daemons/greeter",
|
|
"Restart=always",
|
|
"WantedBy=multi-user.target",
|
|
} {
|
|
if !strings.Contains(unit, want) {
|
|
t.Fatalf("the unit does not say %q:\n%s", want, unit)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **Generated whole and saying so.** Every managed file on a machine carries this, because an edit
|
|
// that survives until the next declaration and then vanishes is worse than one that is refused.
|
|
func TestTheUnitSaysItIsTheMeshs(t *testing.T) {
|
|
unit := unitFor(aProcess())
|
|
if !strings.HasPrefix(unit, "#") || !strings.Contains(unit, "Do not edit") {
|
|
t.Fatalf("the unit does not say it is generated:\n%s", unit)
|
|
}
|
|
}
|
|
|
|
// **Deterministic, because the unit is half the daemon's identity.** Environment held in a map
|
|
// would be written in Go's iteration order, so every apply would see a different unit and call an
|
|
// unchanged daemon changed — restarting it on every declaration for ever.
|
|
func TestTheUnitIsTheSameEveryTime(t *testing.T) {
|
|
first := unitFor(aProcess())
|
|
for i := 0; i < 20; i++ {
|
|
if again := unitFor(aProcess()); again != first {
|
|
t.Fatalf("two renderings of one daemon differ:\n%s\n---\n%s", first, again)
|
|
}
|
|
}
|
|
// And sorted, so the order is a decision rather than luck.
|
|
if strings.Index(first, "A_FIRST") > strings.Index(first, "MESH_NODE") {
|
|
t.Fatalf("environment is not in a stable order:\n%s", first)
|
|
}
|
|
}
|
|
|
|
// **Two daemons from one bundle differing only in their command are different daemons.** Tracking
|
|
// the digest alone would call the second one unchanged and leave the first one running.
|
|
func TestAProcesssIdentityIncludesHowItIsRun(t *testing.T) {
|
|
one := aProcess()
|
|
two := aProcess()
|
|
two.Run = []string{"node", "other.js"}
|
|
if unitFor(one) == unitFor(two) {
|
|
t.Fatal("two daemons with different commands render one unit, so a change would be missed")
|
|
}
|
|
}
|
|
|
|
// A process that runs as somebody says so, and one that does not says nothing — rather than naming
|
|
// root explicitly, which would be a claim the mesh does not need to make.
|
|
func TestAProcessRunsAsWhoItSaid(t *testing.T) {
|
|
as := aProcess()
|
|
as.User = "greeter"
|
|
if !strings.Contains(unitFor(as), "User=greeter") {
|
|
t.Fatalf("the unit does not run as the user it named:\n%s", unitFor(as))
|
|
}
|
|
if strings.Contains(unitFor(aProcess()), "User=") {
|
|
t.Fatalf("a process that named no user had one written for it:\n%s", unitFor(aProcess()))
|
|
}
|
|
|
|
// A process that yields (ADR 0553) runs in the node's one yielding slice at nice 19; the slice holds the idle weight
|
|
// and the quota for its share of the machine's processors; one that does not yield has none of it.
|
|
defer func(n func() int) { machineCPUs = n }(machineCPUs)
|
|
for _, c := range []struct {
|
|
cpus int
|
|
y *declaration.Yields
|
|
unit, slice, not []string
|
|
}{
|
|
{24, &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50},
|
|
[]string{"Slice=meshyield.slice\n", "Nice=19\n", "IOSchedulingPriority=7\n", "50%"},
|
|
[]string{"CPUWeight=idle\n", "IOWeight=1\n", "CPUQuota=1200%\n"}, []string{"CPUQuota=", "CPUWeight="}},
|
|
{1, &declaration.Yields{CPUShare: 25}, []string{"Slice=meshyield.slice\n"}, []string{"CPUQuota=100%\n"},
|
|
[]string{"Nice="}},
|
|
{24, nil, nil, nil, []string{"Slice=", "Nice=", "CPUQuota="}},
|
|
} {
|
|
machineCPUs = func() int { return c.cpus }
|
|
p := aProcess()
|
|
p.Yields = c.y
|
|
unit := unitFor(p)
|
|
for _, w := range c.unit {
|
|
if !strings.Contains(unit, w) {
|
|
t.Errorf("yields %+v: the unit does not say %q:\n%s", c.y, w, unit)
|
|
}
|
|
}
|
|
for _, n := range c.not {
|
|
if strings.Contains(unit, n) {
|
|
t.Errorf("yields %+v: the unit says %q:\n%s", c.y, n, unit)
|
|
}
|
|
}
|
|
if c.y == nil {
|
|
continue
|
|
}
|
|
slice := yieldSlice(c.y)
|
|
for _, w := range c.slice {
|
|
if !strings.Contains(slice, w) {
|
|
t.Errorf("yields %+v on %d processors: the slice does not say %q:\n%s", c.y, c.cpus, w, slice)
|
|
}
|
|
}
|
|
}
|
|
// The slice is placed beside the units, once, and again only when it differs.
|
|
was := unitDir
|
|
unitDir = t.TempDir()
|
|
defer func() { unitDir = was }()
|
|
p := aProcess()
|
|
p.Yields = &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50}
|
|
for i, want := range []bool{true, false} {
|
|
if wrote, err := placeYieldSlice(p); err != nil || wrote != want {
|
|
t.Errorf("placing %d: wrote %v (%v), want %v", i, wrote, err, want)
|
|
}
|
|
}
|
|
if b, err := os.ReadFile(filepath.Join(unitDir, "meshyield.slice")); err != nil || !strings.Contains(string(b), "[Slice]") {
|
|
t.Errorf("the slice file: %q %v", b, err)
|
|
}
|
|
}
|
|
|
|
// **Three modes, one kind.** A scheduled process is a timer plus a unit that finishes, not a unit
|
|
// that stays up — and the difference has to be in what is written, or a schedule becomes a second
|
|
// copy running continuously between fires.
|
|
func TestAScheduledProcessRunsOnItsCadenceRatherThanContinuously(t *testing.T) {
|
|
every := aProcess()
|
|
every.Schedule = "0 3 * * *"
|
|
|
|
unit := unitFor(every)
|
|
if strings.Contains(unit, "Restart=always") {
|
|
t.Fatalf("a scheduled process is restarted whenever it exits, so it never stops:\n%s", unit)
|
|
}
|
|
if !strings.Contains(unit, "Type=oneshot") {
|
|
t.Fatalf("a scheduled process is not a step that finishes:\n%s", unit)
|
|
}
|
|
|
|
timer := timerFor(every)
|
|
if !strings.Contains(timer, "OnCalendar=") {
|
|
t.Fatalf("a scheduled process has no cadence:\n%s", timer)
|
|
}
|
|
// A fire missed while the machine was off happens when it returns, rather than being skipped —
|
|
// the difference between a machine that was down and a schedule that quietly stopped.
|
|
if !strings.Contains(timer, "Persistent=true") {
|
|
t.Fatalf("a missed fire is skipped silently:\n%s", timer)
|
|
}
|
|
}
|
|
|
|
// Five-field cron becomes what a timer reads, rather than the host waking to decide.
|
|
func TestACronBecomesATimersCalendar(t *testing.T) {
|
|
for cron, want := range map[string]string{
|
|
"0 3 * * *": "*-*-* 3:0:00",
|
|
"30 4 1 * *": "*-*-1 4:30:00",
|
|
"0 0 * * mon": "mon *-*-* 0:0:00",
|
|
} {
|
|
if got := calendarFor(cron); got != want {
|
|
t.Fatalf("%q became %q rather than %q", cron, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// And the long-running mode is unchanged by any of it: it stays up and comes back.
|
|
func TestAProcessThatStaysUpIsStillRestartedWhenItExits(t *testing.T) {
|
|
unit := unitFor(aProcess())
|
|
if !strings.Contains(unit, "Restart=always") {
|
|
t.Fatalf("a process that should stay up is not restarted when it exits:\n%s", unit)
|
|
}
|
|
if strings.Contains(unit, "Type=oneshot") {
|
|
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
|
}
|
|
}
|
|
|
|
// **A unit file reinterprets a value in three ways nothing else does**, and a module's environment
|
|
// routinely contains all three — a generated password is arbitrary bytes.
|
|
func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
|
|
// A percent begins a specifier: %H is the hostname. A password containing one would be
|
|
// silently replaced, failing as an authentication error nobody can explain from the
|
|
// declaration.
|
|
percent := aProcess()
|
|
percent.Env = map[string]string{"PASSWORD": "a%Hb"}
|
|
if !strings.Contains(unitFor(percent), "%%H") {
|
|
t.Fatalf("a percent was left as a systemd specifier:\n%s", unitFor(percent))
|
|
}
|
|
|
|
// Whitespace separates assignments: unquoted, K=a b sets K to "a" and reads "b" as another.
|
|
spaced := aProcess()
|
|
spaced.Env = map[string]string{"GREETING": "hello there"}
|
|
if !strings.Contains(unitFor(spaced), `"GREETING=hello there"`) {
|
|
t.Fatalf("a value with a space was not quoted:\n%s", unitFor(spaced))
|
|
}
|
|
|
|
// A quote would end the quoting early, and what follows would be read as unit syntax.
|
|
quoted := aProcess()
|
|
quoted.Env = map[string]string{"TOKEN": `a"b`}
|
|
line := ""
|
|
for _, l := range strings.Split(unitFor(quoted), "\n") {
|
|
if strings.HasPrefix(l, "Environment=") {
|
|
line = l
|
|
}
|
|
}
|
|
if !strings.Contains(line, `\"`) {
|
|
t.Fatalf("a quote was not escaped, so the value ends early: %s", line)
|
|
}
|
|
}
|
|
|
|
func TestAnUndeclaredProcessIsRemovedWithItsUnitAndBundle(t *testing.T) {
|
|
// novox/hq ADR 0118: a process's unit and bundle are the host's own, so they go with the
|
|
// declaration. Before, there was no way to remove a process at all, and one left undeclared
|
|
// failed every apply on its node.
|
|
units, bundles := t.TempDir(), t.TempDir()
|
|
wasUnits, wasBundles := unitDir, daemonRoot
|
|
unitDir, daemonRoot = units, bundles
|
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
|
|
|
for _, f := range []string{"mesh-job.service", "mesh-job.timer"} {
|
|
if err := os.WriteFile(filepath.Join(units, f), []byte("[Unit]\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(bundles, "mesh-job", "bin"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var commands []string
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
commands = append(commands, strings.Join(args, " "))
|
|
return "", nil
|
|
}
|
|
known := store.State{Resources: []store.Applied{{ID: "p", Type: "process", Target: "mesh-job"}}}
|
|
d := parse(t, `{"declaration":1,"resources":[
|
|
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
|
|
]}`)
|
|
report, after, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("an undeclared process failed the apply: %v", err)
|
|
}
|
|
joined := strings.Join(commands, "; ")
|
|
timer, service := strings.Index(joined, "disable --now mesh-job.timer"), strings.Index(joined, "disable --now mesh-job.service")
|
|
if timer < 0 || service < 0 || timer > service {
|
|
t.Errorf("the timer and the unit were not stopped, timer first: %s", joined)
|
|
}
|
|
if !strings.Contains(joined, "daemon-reload") {
|
|
t.Errorf("the service manager was not told its units changed: %s", joined)
|
|
}
|
|
for _, gone := range []string{filepath.Join(units, "mesh-job.service"), filepath.Join(units, "mesh-job.timer"), filepath.Join(bundles, "mesh-job")} {
|
|
if _, err := os.Stat(gone); !os.IsNotExist(err) {
|
|
t.Errorf("%s is still there", gone)
|
|
}
|
|
}
|
|
if o := outcomeOf(report, "p"); o.Action != "removed" {
|
|
t.Errorf("outcome %+v, want removed", o)
|
|
}
|
|
if _, still := after.Find("p"); still {
|
|
t.Error("the host still believes it owns the process")
|
|
}
|
|
|
|
// Again, with everything already gone: forgotten, not an error.
|
|
_, _, err = Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
|
if err != nil {
|
|
t.Errorf("removing a process that is already gone failed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAProcessRecordedUnderAPathlikeNameIsRefusedNotRemoved(t *testing.T) {
|
|
// filepath.Join(daemonRoot, "..") is the mesh's own directory, and removal deletes what that
|
|
// names, whole. A record is what some host wrote, perhaps under looser rules than today's, so
|
|
// the removal holds the name to the declaration's rule again (novox/hq ADR 0118).
|
|
root := t.TempDir()
|
|
bundles := filepath.Join(root, "daemons")
|
|
wasUnits, wasBundles := unitDir, daemonRoot
|
|
unitDir, daemonRoot = t.TempDir(), bundles
|
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
|
precious := filepath.Join(root, "state.json")
|
|
if err := os.MkdirAll(filepath.Join(bundles, "other"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(precious, []byte("{}"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"..", ".", "", "-x"} {
|
|
known := store.State{Resources: []store.Applied{{ID: "p", Type: "process", Target: name}}}
|
|
d := parse(t, `{"declaration":1,"resources":[
|
|
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
|
|
]}`)
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, noServices, nil, nil); err == nil {
|
|
t.Errorf("a process recorded as %q was removed by name", name)
|
|
}
|
|
for _, still := range []string{precious, filepath.Join(bundles, "other")} {
|
|
if _, err := os.Stat(still); err != nil {
|
|
t.Fatalf("removing a process recorded as %q took %s with it", name, still)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/210: the node's runtime was re-created — and restarted — on every reconcile,
|
|
// because the host did not find what it wrote for a process the cycle before. Applying the same
|
|
// process declaration twice must do no work the second time.
|
|
func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
|
|
units, bundles := t.TempDir(), t.TempDir()
|
|
wasUnits, wasBundles := unitDir, daemonRoot
|
|
unitDir, daemonRoot = units, bundles
|
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
|
|
|
body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"})
|
|
var commands []string
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
commands = append(commands, name+" "+strings.Join(args, " "))
|
|
return "", nil
|
|
}
|
|
d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
|
|
`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
|
|
|
|
first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
|
|
t.Fatalf("first apply: %+v, want created", o)
|
|
}
|
|
rec, ok := state.Find("node-tools.runtime")
|
|
if !ok || rec.Wrote == "" {
|
|
t.Fatalf("the host did not record what it wrote for the process: %+v", rec)
|
|
}
|
|
commands = nil
|
|
again, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if o := outcomeOf(again, "node-tools.runtime"); o.Action != "unchanged" {
|
|
t.Errorf("second apply: %+v, want unchanged", o)
|
|
}
|
|
for _, c := range commands {
|
|
if strings.Contains(c, "restart") {
|
|
t.Errorf("the second apply restarted the process: %v", commands)
|
|
}
|
|
}
|
|
// And the record survives an unchanged apply: the third cycle is unchanged too. This is the
|
|
// cycle the live mesh showed — created, unchanged, created — before the record was carried.
|
|
if rec, _ := state.Find("node-tools.runtime"); rec.Wrote == "" {
|
|
t.Fatalf("an unchanged apply dropped the digest from the record: %+v", rec)
|
|
}
|
|
commands = nil
|
|
third, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if o := outcomeOf(third, "node-tools.runtime"); o.Action != "unchanged" {
|
|
t.Errorf("third apply: %+v, want unchanged", o)
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0193: a bundle compiled to a binary runs itself — `./name` is that file in the
|
|
// process's own unpacked bundle, made absolute because the service manager takes nothing relative.
|
|
func TestAProcessRunsItsOwnBundlesBinary(t *testing.T) {
|
|
unit := unitFor(&declaration.Process{Name: "node-tools", Run: []string{"./node-tools", "serve"}})
|
|
if !strings.Contains(unit, "ExecStart="+filepath.Join(daemonRoot, "node-tools", "node-tools")+" serve\n") {
|
|
t.Errorf("the binary is not run from its bundle:\n%s", unit)
|
|
}
|
|
other := unitFor(&declaration.Process{Name: "x", Run: []string{"node", "src/main.js"}})
|
|
if !strings.Contains(other, "ExecStart=node src/main.js\n") {
|
|
t.Errorf("a command found on the path was changed:\n%s", other)
|
|
}
|
|
}
|