diff --git a/internal/apply/filled.go b/internal/apply/filled.go new file mode 100644 index 0000000..a0d750f --- /dev/null +++ b/internal/apply/filled.go @@ -0,0 +1,203 @@ +package apply + +import ( + "context" + "fmt" + "slices" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// WriteFilled writes what the provisioner's client filled into one file the declaration names as filled (novox/hq +// ADR 0349 §4), and makes what reads the file take the change, exactly as an apply that changed it would. +// +// **The declared file says everything but the content.** Its path, mode, owner and who owns it at runtime are the +// declaration's; the client's request carries the content alone. So the file is applied as the declared one with +// that content in place of the declaration's: an owned file whole through applyFile, a seed only when absent, a +// managed file's keys through applyInto or its block through applyBlock — the same guards on where nothing is +// placed, the same Keep of an original found there, the same record. What the declaration seals or fills with +// `${secret:}` the client has already filled, so the content is written as it came; a file that carried a secret +// keeps the mode a secret is written with when the declaration gives none. +// +// **Then what reads it, in the apply's order** (afterWhatTheyRead): a service naming it under restart-on is +// restarted and under reload-on reloaded, a process or container naming it under restart-on is restarted, and a +// running container that reads it at creation — an env-file, a file mounted into it — is recreated, because what +// it read is recorded by digest and this write changed the digest. Each of those is applied and recorded as an +// apply does, and what moves goes on moving what names it. Nothing else of the declaration is touched. +// +// The caller holds the apply's lock: a write and an apply never write one file at once. log is said the path, the +// action and the digest, never the content. +func WriteFilled( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + path string, + content []byte, + run Runner, + log func(string), + keep Keep, + windows *Windows, +) (report Report, _ store.State, _ error) { + if log == nil { + log = func(string) {} + } + if !d.Fills(path) { + return report, known, &Error{Resource: path, Err: fmt.Errorf("this node's declaration does not name %s "+ + "as a file the provisioner fills, so nothing was written", path)} + } + var declared []*declaration.File + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && f.Path == path { + declared = append(declared, f) + } + } + if len(declared) != 1 { + // The parser refuses a filled path no file declares; two at one path would need the request to say whose + // keys or block it carries, and it carries the content only. + return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+ + "is written for exactly one, so nothing was written", len(declared), path)} + } + file := filledAs(declared[0], content) + if held, isHeld := known.HeldAt(file.ID); isHeld { + return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+ + "yet taken (%s), so nothing was written", path, held.Why)} + } + + in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log), + groups: groupsWanted(d.Resources), handed: Handed{}} + refreshRuntimeRoots(ctx) + in.refused = refusedPlaces(d.Resources) + in.agentHomes = rootNeverHomes(d) + in.dirsBefore = map[string]bool{} + for _, r := range d.Resources { + in.declares[r.Identity()] = declaredDigest(r) + } + changed := map[string]bool{} + + // The file, as the apply loop applies it. + was, _ := known.Find(file.ID) + var keepFound Keep + if (was.ID == "" || was.Target != file.Path) && !known.Recorded(string(declaration.TypeFile), file.Path) { + keepFound = keep + } + outcome, err := applyOne(ctx, sys, file, run, changed, in, was, nil, keepFound) + if err != nil { + log(fmt.Sprintf(" failed %s (%s): %v", file.ID, file.Path, err)) + return report, known, &Error{Resource: file.ID, Err: err, Done: report} + } + recordKeeping(&known, file, outcome, was) + report.Outcomes = append(report.Outcomes, outcome) + if !moved(outcome) { + return report, known, nil + } + changed[file.ID] = true + line := fmt.Sprintf(" %s %s (%s), filled: %s", outcome.Action, file.ID, file.Path, outcome.wrote) + if outcome.Detail != "" { + line += ": " + outcome.Detail + } + log(line) + + // What reads it, in the apply's order, each moving what names it in turn. + var failures []*Error + for _, r := range afterWhatTheyRead(d.Resources) { + if r.Identity() == file.ID || !readsChange(r, path, changed) { + continue + } + if _, isHeld := known.HeldAt(r.Identity()); isHeld { + log(fmt.Sprintf(" held %s (%s): held as found for a module not yet taken, so it was not restarted "+ + "for %s", r.Identity(), r.Target(), file.ID)) + continue + } + was, _ := known.Find(r.Identity()) + o, err := applyOne(ctx, sys, r, run, changed, in, was, nil, nil) + if err != nil { + if o.found != nil { + known.KeepFound(r.Identity(), store.OriginDeclared, *o.found) + } + failures = append(failures, &Error{Resource: r.Identity(), Err: err}) + log(fmt.Sprintf(" failed %s (%s): %v", r.Identity(), o.Target, err)) + continue + } + if o.Action == "waiting" { + if o.found != nil { + known.KeepFound(r.Identity(), store.OriginDeclared, *o.found) + } + report.Outcomes = append(report.Outcomes, o) + log(fmt.Sprintf(" waiting %s (%s): %s", o.ID, o.Target, o.Detail)) + continue + } + recordKeeping(&known, r, o, was) + report.Outcomes = append(report.Outcomes, o) + if o.busy != nil { + report.Busy = append(report.Busy, *o.busy) + } + if moved(o) { + changed[r.Identity()] = true + line := fmt.Sprintf(" %s %s (%s)", o.Action, o.ID, o.Target) + if o.Detail != "" { + line += ": " + o.Detail + } + log(line) + } else if o.Action != "unchanged" { + log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail)) + } + } + if len(failures) > 0 { + first := failures[0] + first.Done = report + first.Others = len(failures) - 1 + return report, known, first + } + return report, known, nil +} + +// filledAs is the declared file with the filled content in place of the declaration's. +func filledAs(declared *declaration.File, content []byte) *declaration.File { + f := *declared + f.Content, f.Sealed, f.Bytes, f.Secrets = string(content), "", "", nil + if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) { + // Written as the declaration would have written it: a file carrying a credential is not world-readable. + f.Mode = "0600" + } + return &f +} + +// readsChange reports whether applying r is what makes it take a change of what changed: a service's restart-on or +// reload-on, a process's or container's restart-on, or a running container that reads path at creation. +func readsChange(r declaration.Resource, path string, changed map[string]bool) bool { + switch res := r.(type) { + case *declaration.Service: + return len(restartedBy(res.RestartOn, changed)) > 0 || len(restartedBy(res.ReloadOn, changed)) > 0 + case *declaration.Process: + return len(restartedBy(res.RestartOn, changed)) > 0 + case *declaration.Container: + if len(restartedBy(res.RestartOn, changed)) > 0 { + return true + } + if res.RunOnce || res.Schedule != "" { + return false + } + if slices.Contains(res.EnvFile, path) { + return true + } + for _, v := range res.Volumes { + if mountSource(v) == path { + return true + } + } + } + return false +} + +// recordKeeping records r as applied, carrying where the original it replaced was kept from its record when this +// write kept none — for as long as the resource is recorded at the same target, as the apply loop carries it. +func recordKeeping(known *store.State, r declaration.Resource, o Outcome, was store.Applied) { + kept, keptMode, keptOwner := o.kept, o.keptMode, o.keptOwner + if kept == "" && was.Target == o.Target { + kept, keptMode, keptOwner = was.Kept, was.KeptMode, was.KeptOwner + } + recordApplied(known, store.OriginDeclared, r, o, kept, keptMode, keptOwner) +} diff --git a/internal/apply/filled_test.go b/internal/apply/filled_test.go new file mode 100644 index 0000000..b2e9003 --- /dev/null +++ b/internal/apply/filled_test.go @@ -0,0 +1,247 @@ +package apply + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): the provisioner's client has the node-engine write a file it filled, +// and the node-engine writes it as an apply of the declared file with that content would — through applyFile, +// applyInto or applyBlock by who owns it at runtime — and makes what reads it take the change as an apply does. + +// filledDecl is a declaration naming path as filled, with the file's fields after id, type and path, and more +// resources after it. +func filledDecl(t *testing.T, path, fields, more string) *declaration.Declaration { + t.Helper() + if more != "" { + more = "," + more + } + return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner", + "filled_files":[%q],"resources":[{"id":"m.conf","type":"file","path":%q,%s}%s]}`, path, path, fields, more)) +} + +func TestAWriteOfAPathNotFilledIsRefused(t *testing.T) { + dir := t.TempDir() + filled := filepath.Join(dir, "filled.conf") + other := filepath.Join(dir, "other.conf") + d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[ + {"id":"m.conf","type":"file","path":%q,"content":"x\n","owner-at-runtime":"owned"}, + {"id":"m.other","type":"file","path":%q,"content":"y\n"} + ]}`, filled, filled, other)) + for _, path := range []string{other, filepath.Join(dir, "nothing.conf")} { + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("evil\n"), + noServices, nil, nil, nil) + if err == nil || !strings.Contains(err.Error(), "does not name") { + t.Errorf("%s: a write of a path the declaration does not fill was not refused: %v", path, err) + } + if _, err := os.Stat(path); err == nil && path != other { + t.Errorf("%s was written", path) + } + if len(state.Resources) != 0 { + t.Errorf("a refused write recorded something: %+v", state.Resources) + } + } + if _, err := os.Stat(other); err == nil { + t.Errorf("%s was written", other) + } +} + +// Each ownership is written by the write verb exactly as an apply of the declared file with the same content: the +// same action, the same bytes on disk, the same mode. +func TestAFilledFileIsWrittenAsItsOwnershipSays(t *testing.T) { + for _, c := range []struct { + name, fields, content, before, action string + }{ + {"owned, absent", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", "", "created"}, + {"owned, over what is there", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", + "port = 1\n", "updated"}, + {"seed, absent", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", "", "created"}, + {"seed, present", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", + "user default on\nuser app-one on\n", "kept"}, + {"managed json", `"into":"json","content":"{}","owner-at-runtime":"managed"`, + `{"insecure-registries":["10.42.0.1:5000"]}`, machinesOwn, "updated"}, + {"managed block", `"into":"block","content":"x\n","owner-at-runtime":"managed"`, "10.42.0.1 ace\n", + workstationHosts, "updated"}, + } { + t.Run(c.name, func(t *testing.T) { + at := func() string { + path := filepath.Join(t.TempDir(), "the-file") + if c.before != "" { + if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil { + t.Fatal(err) + } + } + return path + } + written := at() + report, state, err := WriteFilled(context.Background(), archHost(t), filledDecl(t, written, c.fields, ""), + store.State{}, written, []byte(c.content), noServices, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + applied := at() + withContent := strings.Replace(c.fields, fmt.Sprintf(`"content":%q`, contentOf(t, c.fields)), + fmt.Sprintf(`"content":%q`, c.content), 1) + want, _, err := Apply(context.Background(), archHost(t), filledDecl(t, applied, withContent, ""), + store.State{}, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != c.action || got != want.Outcomes[0].Action { + t.Errorf("written %s; an apply of the same content %s; want %s", got, want.Outcomes[0].Action, c.action) + } + if got, apply := readText(t, written), readText(t, applied); got != apply { + t.Errorf("written, the file holds\n%q\nan apply of the same content leaves\n%q", got, apply) + } + if a, b := modeAt(t, written), modeAt(t, applied); a != b { + t.Errorf("written mode %o, an apply's %o", a, b) + } + if rec, ok := state.Find("m.conf"); !ok || rec.Target != written || rec.Origin != store.OriginDeclared { + t.Errorf("the write is not recorded as the file's: %+v", rec) + } + }) + } +} + +func contentOf(t *testing.T, fields string) string { + t.Helper() + var holder struct { + Content string `json:"content"` + } + if err := json.Unmarshal([]byte("{"+fields+"}"), &holder); err != nil { + t.Fatal(err) + } + return holder.Content +} + +func modeAt(t *testing.T, path string) os.FileMode { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return info.Mode().Perm() +} + +// A file that carried a secret keeps the mode a secret is written with: the client filled the value, and the file +// is no more readable for it. +func TestAFilledFileThatCarriedASecretIsNotWorldReadable(t *testing.T) { + path := filepath.Join(t.TempDir(), "token.env") + d := filledDecl(t, path, `"content":"TOKEN=${secret:token}\n","secrets":{"token":"c2VhbGVk"},"owner-at-runtime":"owned"`, "") + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("TOKEN=abc\n"), + noServices, nil, nil, nil); err != nil { + t.Fatal(err) + } + if m := modeAt(t, path); m != 0o600 { + t.Errorf("a filled file that carried a secret is mode %o", m) + } +} + +func TestAWriteUnderReloadOnReloadsItsProgramOnce(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.conf") + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, + `{"id":"m.daemon","type":"service","unit":"m.service","state":"running","reload-on":["m.conf"]}, + {"id":"m.other","type":"service","unit":"other.service","state":"running"}`) + var commands []string + var said []string + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"), + recordingServices(&commands), func(line string) { said = append(said, line) }, nil, nil) + if err != nil { + t.Fatal(err) + } + if n := count(commands, "systemctl reload m.service"); n != 1 { + t.Errorf("the program was reloaded %d times; commands were %v", n, commands) + } + joined := strings.Join(commands, "\n") + if strings.Contains(joined, "restart m.service") || strings.Contains(joined, "stop m.service") { + t.Errorf("the program was restarted, not reloaded: %v", commands) + } + if strings.Contains(joined, "other.service") { + t.Errorf("a service that does not read the file was touched: %v", commands) + } + if _, ok := state.Find("m.daemon"); !ok { + t.Errorf("the reloaded service was not recorded") + } + // The log says the path and the digest, never the content. + for _, line := range said { + if strings.Contains(line, "level = 1") { + t.Errorf("the content was said: %q", line) + } + } + if !strings.Contains(strings.Join(said, "\n"), digestOf("level = 1\n")) { + t.Errorf("the digest was not said: %q", said) + } + + // The same content again moves nothing, and reloads nothing. + commands = nil + report, _, err := WriteFilled(context.Background(), archHost(t), d, state, path, []byte("level = 1\n"), + recordingServices(&commands), nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if report.Outcomes[0].Action != "unchanged" || count(commands, "reload") != 0 { + t.Errorf("an unchanged write reloaded: %v %+v", commands, report.Outcomes) + } +} + +func TestAWriteUnderRestartOnRestartsItsProgram(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.conf") + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, + `{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`) + var commands []string + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"), + recordingServices(&commands), nil, nil, nil); err != nil { + t.Fatal(err) + } + // A restart is a stop and a start, once. + stops, starts := count(commands, "systemctl stop m.service"), count(commands, "systemctl start m.service") + if stops != 1 || starts != 1 { + t.Errorf("the program was stopped %d and started %d times; commands were %v", stops, starts, commands) + } +} + +// A running container that reads the file at creation is recreated, as an apply recreates it for a changed file. +func TestAWriteOfAFileAContainerReadsRecreatesIt(t *testing.T) { + env := filepath.Join(t.TempDir(), "forge.env") + d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`, + `{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`) + m := &machine{containers: map[string]*fakeContainer{}} + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"), + m.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !m.did("docker run") { + t.Fatalf("the container was not created: %v", m.asked) + } + m.asked = nil + report, _, err := WriteFilled(context.Background(), archHost(t), d, state, env, []byte("PORT=5433\n"), m.run, + nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !m.removed("forge") || !m.did("docker run") { + t.Fatalf("the container kept what it read before the write: %v", m.asked) + } + if o := outcomeOf(report, "forge.server"); o.Action != "updated" || !strings.Contains(o.Detail, env) { + t.Errorf("the recreate did not say which file changed: %+v", o) + } +} + +func count(lines []string, prefix string) int { + n := 0 + for _, l := range lines { + if strings.Contains(l, prefix) { + n++ + } + } + return n +}