// Package apply makes a machine match a declaration. // // Three properties, each following a recorded decision, and each of them the difference // between this and a script that writes files: // // - A failed step fails the apply (novox/hq ADR 0008). Not "logs and continues": a partial // apply that reports success is the mesh's most expensive shape. // - Every applier READS BACK. Setting a value is not evidence the value took. // - What was applied is recorded after it works, never before (ADR 0035). A failed apply // leaves the machine in whatever state it reached, and nothing must claim otherwise. package apply import ( "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strconv" "strings" "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // Runner executes a command. The real one is used everywhere outside unit tests; behaviour // against a real system is tested alongside rather than mocked (novox/hq ADR 0034). type Runner func(ctx context.Context, name string, args ...string) (string, error) // Outcome is what happened to one resource. type Outcome struct { ID string `json:"id"` Type string `json:"type"` Target string `json:"target"` Action string `json:"action"` // created · updated · unchanged · removed Detail string `json:"detail,omitempty"` } // Report is what an apply did, in the order it did it. type Report struct { Outcomes []Outcome `json:"outcomes"` } // Changed reports whether anything about the machine actually moved. An apply that changed // nothing is the ordinary steady state, and saying so is not the same as saying it failed. func (r Report) Changed() bool { for _, o := range r.Outcomes { if o.Action != "unchanged" { return true } } return false } // Error is a failure part-way through, carrying what had already been done. // // The outcomes matter as much as the message: the machine is in whatever state the apply // reached, and the only honest thing to hand back is the list of what did happen. type Error struct { Resource string Err error Done Report } func (e *Error) Error() string { return fmt.Sprintf("applying %q: %v\n\n%d resource(s) were applied before this and remain; "+ "the machine is in whatever state that left it.", e.Resource, e.Err, len(e.Done.Outcomes)) } func (e *Error) Unwrap() error { return e.Err } // Apply makes the machine match the declaration, and returns what it did. // // Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration // while another arrives at the same path is an ordinary rename: removing afterwards would // delete the file that had just been written. Removing first risks losing the old state if the // apply then fails — a recovery concern, where the other is a correctness one. func Apply( ctx context.Context, d *declaration.Declaration, known store.State, run Runner, log func(string), ) (Report, store.State, error) { if log == nil { log = func(string) {} } report := Report{} declared := map[string]bool{} for _, r := range d.Resources { declared[r.ID] = true } for _, orphan := range known.Orphans(declared) { if err := remove(ctx, orphan, run); err != nil { return report, known, &Error{Resource: orphan.ID, Err: err, Done: report} } known.Forget(orphan.ID) report.Outcomes = append(report.Outcomes, Outcome{ ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "removed", Detail: "no longer declared", }) log(fmt.Sprintf(" removed %s (%s)", orphan.ID, orphan.Target)) } for _, resource := range d.Resources { outcome, err := applyOne(ctx, resource, run) if err != nil { return report, known, &Error{Resource: resource.ID, Err: err, Done: report} } // Only now. The record follows the fact, never leads it. known.Record(store.Applied{ ID: resource.ID, Type: string(resource.Type), Target: outcome.Target, AppliedAt: time.Now().UTC(), }) report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { log(fmt.Sprintf(" %s %s (%s)", outcome.Action, outcome.ID, outcome.Target)) } } return report, known, nil } func applyOne(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) { switch r.Type { case declaration.TypeDirectory: return applyDirectory(r) case declaration.TypeFile: return applyFile(r) case declaration.TypeService: return applyService(ctx, r, run) default: // Unreachable: the declaration refused this already. Present because "unreachable" // stops being true the moment someone adds a type and forgets this switch. return Outcome{}, fmt.Errorf("no applier for type %q", r.Type) } } func modeOf(spec string, fallback os.FileMode) (os.FileMode, error) { if spec == "" { return fallback, nil } parsed, err := strconv.ParseUint(spec, 8, 32) if err != nil { return 0, fmt.Errorf("mode %q: %w", spec, err) } return os.FileMode(parsed), nil } func applyDirectory(r declaration.Resource) (Outcome, error) { out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Path} mode, err := modeOf(r.Mode, 0o755) if err != nil { return out, err } before, err := os.Stat(r.Path) existed := err == nil if err != nil && !errors.Is(err, os.ErrNotExist) { return out, err } if existed && !before.IsDir() { return out, fmt.Errorf("%s exists and is not a directory", r.Path) } if !existed { if err := os.MkdirAll(r.Path, mode); err != nil { return out, err } } // Set explicitly even when it existed: MkdirAll applies the mode only on creation, and a // permission set at creation is not a permission maintained — a lesson this repository // already paid for once, with world-readable environment files. if err := os.Chmod(r.Path, mode); err != nil { return out, err } // Read back. after, err := os.Stat(r.Path) if err != nil { return out, fmt.Errorf("made %s and cannot stat it: %w", r.Path, err) } if !after.IsDir() { return out, fmt.Errorf("%s is not a directory after applying", r.Path) } if after.Mode().Perm() != mode.Perm() { return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, after.Mode().Perm(), mode.Perm()) } out.Action = "unchanged" if !existed { out.Action = "created" } else if before.Mode().Perm() != mode.Perm() { out.Action = "updated" out.Detail = fmt.Sprintf("mode %o to %o", before.Mode().Perm(), mode.Perm()) } return out, nil } func applyFile(r declaration.Resource) (Outcome, error) { out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Path} mode, err := modeOf(r.Mode, 0o644) if err != nil { return out, err } existing, readErr := os.ReadFile(r.Path) existed := readErr == nil if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { return out, readErr } var beforeMode os.FileMode if existed { if info, err := os.Stat(r.Path); err == nil { beforeMode = info.Mode().Perm() } } contentSame := existed && string(existing) == r.Content modeSame := existed && beforeMode == mode.Perm() if !contentSame { if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { return out, err } if err := writeAtomically(r.Path, []byte(r.Content), mode); err != nil { return out, err } } else if !modeSame { if err := os.Chmod(r.Path, mode); err != nil { return out, err } } // Read back — the file, not the call that wrote it. written, err := os.ReadFile(r.Path) if err != nil { return out, fmt.Errorf("wrote %s and cannot read it back: %w", r.Path, err) } if string(written) != r.Content { return out, fmt.Errorf("%s does not contain what was declared after writing it", r.Path) } info, err := os.Stat(r.Path) if err != nil { return out, err } if info.Mode().Perm() != mode.Perm() { return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, info.Mode().Perm(), mode.Perm()) } switch { case !existed: out.Action = "created" case !contentSame && !modeSame: out.Action = "updated" out.Detail = "content and mode" case !contentSame: out.Action = "updated" out.Detail = "content" case !modeSame: out.Action = "updated" out.Detail = fmt.Sprintf("mode %o to %o", beforeMode, mode.Perm()) default: out.Action = "unchanged" } return out, nil } // writeAtomically writes through a temporary file in the same directory. // // A reader of a managed file must never see half of one. The mesh's own configuration is read // by daemons that reload on change, so a torn write is a service reading a truncated config. func writeAtomically(path string, content []byte, mode os.FileMode) error { tmp, err := os.CreateTemp(filepath.Dir(path), ".mesh-host-*") if err != nil { return err } defer os.Remove(tmp.Name()) if _, err := tmp.Write(content); err != nil { tmp.Close() return err } if err := tmp.Sync(); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } if err := os.Chmod(tmp.Name(), mode); err != nil { return err } return os.Rename(tmp.Name(), path) } func applyService(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) { out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Unit} before, err := serviceState(ctx, r.Unit, run) if err != nil { return out, err } if before == r.State { out.Action = "unchanged" out.Detail = before return out, nil } verb := "start" if r.State == "stopped" { verb = "stop" } if _, err := run(ctx, "systemctl", verb, r.Unit); err != nil { return out, fmt.Errorf("%s %s: %w", verb, r.Unit, err) } // Read back. `systemctl start` returning zero says the transaction was accepted, not that // the unit is running — a unit that starts and immediately dies satisfies the command. after, err := serviceState(ctx, r.Unit, run) if err != nil { return out, err } if after != r.State { return out, fmt.Errorf("%s was asked to be %s and is %s", r.Unit, r.State, after) } out.Action = "updated" out.Detail = before + " to " + after return out, nil } // serviceState reads what the service manager says about a unit. // // Two traps here, and both were hit before this read what it now reads. // // The exit code is not the answer: `is-active` exits non-zero for every state except active — // the same shape as the capability detector reading a degraded init as no init at all. // // And "inactive" does not mean stopped. `systemctl is-active` says "inactive" for a unit that // DOES NOT EXIST exactly as it does for one that is installed and stopped. Declaring a unit // stopped therefore reported success for a unit the host cannot manage at all — absence read // as satisfaction, which is 04-ISSUES/007 wearing a different hat. LoadState is what separates // them, so LoadState is what is read. func serviceState(ctx context.Context, unit string, run Runner) (string, error) { out, _ := run(ctx, "systemctl", "show", unit, "--property=LoadState", "--property=ActiveState") var load, active string for _, line := range strings.Split(out, "\n") { key, value, found := strings.Cut(strings.TrimSpace(line), "=") if !found { continue } switch key { case "LoadState": load = value case "ActiveState": active = value } } switch load { case "": return "", fmt.Errorf("the service manager said nothing about %s", unit) case "not-found": return "", fmt.Errorf( "%s does not exist on this machine. A declaration naming a unit that is not "+ "installed cannot be satisfied, and reporting it stopped would be reporting "+ "absence as success", unit) case "masked": return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit) case "error", "bad-setting": return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load) } switch active { case "active", "activating", "reloading": return "running", nil case "inactive", "failed", "deactivating": return "stopped", nil default: return "", fmt.Errorf( "the service manager reports %s as %q, which is neither running nor stopped", unit, active) } } // remove undoes one resource the host applied and the declaration no longer names. // // Only ever called for something in the store, which is what bounds it: the host is // authoritative over its own footprint and inert everywhere else (novox/hq ADR 0043). func remove(ctx context.Context, a store.Applied, run Runner) error { switch declaration.Type(a.Type) { case declaration.TypeFile, declaration.TypeDirectory: if err := os.RemoveAll(a.Target); err != nil { return err } if _, err := os.Stat(a.Target); !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("%s is still there after removing it", a.Target) } return nil case declaration.TypeService: // A unit that is no longer declared is stopped, not deleted. The host did not install // it and does not own the unit file — only the state it put the unit into. // // A unit that no longer EXISTS is already in the state removal is trying to reach, and // saying so matters: stopping it fails, and a failure here fails the whole apply. A // host holding a record of an uninstalled unit would then be unable to apply anything, // ever, with no way out but editing its state by hand. Removal is idempotent for the // same reason `os.RemoveAll` is. if _, err := serviceState(ctx, a.Target, run); err != nil { if strings.Contains(err.Error(), "does not exist on this machine") { return nil } return err } if _, err := run(ctx, "systemctl", "stop", a.Target); err != nil { return fmt.Errorf("stopping %s: %w", a.Target, err) } return nil default: return fmt.Errorf("no way to remove a %q", a.Type) } } // ExecRunner runs a real command, with stdin closed and output captured. func ExecRunner(ctx context.Context, name string, args ...string) (string, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Stdin = nil out, err := cmd.Output() if err != nil { var exit *exec.ExitError if errors.As(err, &exit) { return string(out), fmt.Errorf("%s exited %d: %s", name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))) } return string(out), fmt.Errorf("%s: %w", name, err) } return string(out), nil }