package apply import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "os" "path/filepath" "strings" "syscall" "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // Keep records the original of a file found on an adopted node, before anything else happens to // it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the // first copy is the one that was there before the mesh. type Keep func(path string, content []byte, mode os.FileMode) (string, error) // KeepIn keeps originals under dir/kept, each named for the path it came from AND for what was in // it, readable by root alone — a predecessor's configuration may carry its credentials. // // **By content as well as path, because a path has more than one original.** A file held, let go // when its module was unassigned, rewritten by the predecessor and found again is a second // original; named by path alone the second copy was silently discarded while the report said it // was kept (novox/hq ADR 0100). The same content at the same path is kept once. func KeepIn(dir string) Keep { return func(path string, content []byte, _ os.FileMode) (string, error) { where := sha256.Sum256([]byte(path)) what := sha256.Sum256(content) kept := filepath.Join(dir, "kept", hex.EncodeToString(where[:])[:12]+"-"+ hex.EncodeToString(what[:])[:12]+"-"+filepath.Base(path)) if _, err := os.Lstat(kept); err == nil { return kept, nil } if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { return "", err } if err := writeAtomically(kept, content, 0o600); err != nil { return "", err } back, err := os.ReadFile(kept) if err != nil || string(back) != string(content) { return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept) } return kept, nil } } // foundBefore is what an adopted apply finds on the machine before it changes anything: each // directory, service unit and container mount source of an untaken module that is present with no // record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a // file's parent directory, a unit file a module writes, a package that brings its unit — and what // the mesh made in this apply was not found. type foundBefore struct { is map[string]bool // trouble is what could not be asked about, by the same key, so the resource that would need // the answer fails rather than proceeding as if the machine had nothing there. trouble map[string]string } func (f foundBefore) has(key string) bool { return f.is[key] } // why is the reason a key could not be settled, and empty when there was none. func (f foundBefore) why(key string) string { return f.trouble[key] } func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State, run Runner) foundBefore { seen := foundBefore{is: map[string]bool{}, trouble: map[string]string{}} if d.Adoption == nil { return seen } cri, asked := "", false for _, r := range d.Resources { if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken { continue } if _, held := known.HeldAt(r.Identity()); held { continue } switch res := r.(type) { case *declaration.Directory: if present(res.Path) && !recordedPath(known, res.Path) { seen.is["path:"+res.Path] = true } case *declaration.Archive: // Unpacking over it, and re-owning it recursively, would change the predecessor's // files. if present(res.Path) && !recordedPath(known, res.Path) { seen.is["path:"+res.Path] = true } case *declaration.Process: // Its unit would be written over and restarted. if !known.Recorded(string(declaration.TypeProcess), res.Name) && present(filepath.Join(unitDir, res.Name+".service")) { seen.is["unit-file:"+res.Name] = true } case *declaration.User: // Its shell and groups would be changed. if !known.Recorded(string(declaration.TypeUser), res.Name) { if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists { seen.is["user:"+res.Name] = true } } case *declaration.Service: if res.Stateless() || known.Recorded(string(declaration.TypeService), res.Unit) { continue } // **Found is a unit somebody put on this machine, or one the machine uses.** // // Where it comes from first: a unit the service manager loads from outside /usr — // /etc/systemd/system or /run/systemd/system — was installed by an administrator, so // it is a predecessor's whatever state it is in, and one deliberately stopped and // disabled must stay that way (novox/hq ADR 0103). // // A unit a package ships, under /usr, is not held by its mere presence: the private // network's own wg-quick@mesh0 is an instance of a template the tunnel package ships, // nothing had ever run it, and holding it kept the private network from ever coming up // (found by the adoption bed). Such a unit is held only if the machine actually uses // it — running, or started at boot. state, err := sys.ServiceState(ctx, run, res.Unit) if err != nil { continue } if from, ok := sys.(unitFiles); ok { if path, err := from.ServiceUnitFile(ctx, run, res.Unit); err == nil && installedByHand(path) { seen.is["unit:"+res.Unit] = true continue } } boot, _ := sys.ServiceBoot(ctx, run, res.Unit) if state == "running" || boot == "enabled" { seen.is["unit:"+res.Unit] = true } case *declaration.Container: if known.Recorded(string(declaration.TypeContainer), res.Name) { continue } for _, v := range res.Volumes { src := mountSource(v) switch { case src == "": case strings.HasPrefix(src, "/"): if !systemPath(src) && present(src) && !recordedPath(known, src) { seen.is["path:"+src] = true } default: if !asked { cri, _ = containerRuntime(ctx, run) asked = true } if cri == "" { continue } if _, err := run(ctx, cri, "volume", "inspect", src); err == nil { seen.is["volume:"+src] = true } else if !absent(err) { seen.trouble["volume:"+src] = fmt.Sprintf( "the container runtime could not say whether the volume %s is here: %v", src, err) } } } } } return seen } // unitFiles is a service manager that can say where it loads a unit from. type unitFiles interface { ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) } // installedByHand is whether a unit file is one somebody put on this machine rather than one a // package ships: anywhere but /usr, where distributions keep what they install. func installedByHand(path string) bool { if path == "" { return false } return !strings.HasPrefix(filepath.Clean(path), "/usr/") } func present(path string) bool { _, err := os.Lstat(path) return err == nil } // recordedPath is whether this host has a record of MAKING something at a path — a directory it // created, a file it wrote, an archive it unpacked. An access record is not one of those: it says // the mesh set permissions on a path it does not own, which is exactly what it does to a path // somebody else's software made, so a path it only has access for is still found (novox/hq ADR 0103). func recordedPath(known store.State, path string) bool { for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive} { if known.Recorded(string(kind), path) { return true } } return false } // systemPath is whether a bind-mount source is the machine's own plumbing — the runtime's socket, // the kernel's filesystems, the devices, the clock — which every machine has and no predecessor's // data lives in. Mounting it shares nothing that was found. func systemPath(src string) bool { clean := filepath.Clean(src) for _, exact := range []string{"/etc/localtime", "/etc/timezone", "/etc/hosts", "/etc/resolv.conf", "/etc/machine-id", "/etc/passwd", "/etc/group"} { if clean == exact { return true } } for _, under := range []string{"/run", "/var/run", "/sys", "/proc", "/dev", "/usr/share/zoneinfo", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/lib/modules", "/usr/lib/modules"} { if clean == under || strings.HasPrefix(clean, under+"/") { return true } } return false } // mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for // an anonymous volume, which mounts nothing that could already be there. func mountSource(mapping string) string { src, _, ok := strings.Cut(mapping, ":") if !ok { return "" } return src } // runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step // sharing a container's namespace. An action with no `in` runs on the machine itself and is not // held for a container: it reaches nothing a predecessor holds by running there, and holding every // action of an untaken module would stop a module preparing itself before its cutover. func runsIn(r declaration.Resource) string { switch res := r.(type) { case *declaration.Action: return res.In case *declaration.Container: if res.RunOnce { if name, ok := strings.CutPrefix(res.Network, "container:"); ok { return name } } } return "" } // heldContainer is what is held under a container's name. func heldContainer(known store.State, name string) (store.Held, bool) { for _, h := range known.Held { if h.Kind == string(declaration.TypeContainer) && h.Target == name { return h, true } } return store.Held{}, false } // replacesNothing is a resource that takes nothing found on the machine from it, so on an adopted // node it is never held and never previewed as replacing what was found: a file written into // (novox/hq ADR 0102), and a service whose unit's lifecycle is the machine's (novox/hq ADR 0117). func replacesNothing(r declaration.Resource) bool { switch res := r.(type) { case *declaration.File: return res.Into != "" case *declaration.Service: return res.Stateless() } return false } // holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and // holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no // record is kept as it is: a file or a container under its name, a directory, a service's unit, // and a container that would mount a path or a volume found there. An action or a run-once step // run inside a held container is held with it. Once held, a resource stays held — changed or gone // — until its module is taken, and it is never recorded as applied, so never removed as an orphan. // // Held is false for a resource to apply as usual. News is whether the hold is new or changed, // which is what is worth a line in the log. func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration, known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool, now time.Time) (held, news bool, out Outcome, err error) { was, already := known.HeldAt(r.Identity()) if in := runsIn(r); in != "" { if container, isHeld := heldContainer(*known, in); isHeld { module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) if !untaken { module = container.Module } h := was if !already { h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now} } h.Module, h.Why = module, "runs in "+in known.RecordHeld(h) out = begin(r) out.Action = "held" out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module) return true, !already, out, nil } } // A file written into replaces nothing that was found, so it is never held (novox/hq ADR // 0102) — and a hold from when it was declared whole must not keep the mesh's keys out. That // hold is released by the apply once the write has worked, not here: a write that fails keeps // it, and with it where the original was kept. A service whose lifecycle is the machine's // replaces nothing either (novox/hq ADR 0117). if replacesNothing(r) { return false, false, out, nil } module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) if !untaken { return false, false, out, nil } why := was.Why isFound := already if !already { switch res := r.(type) { case *declaration.File: if res.Into == "" { if isFound, err = found(ctx, r, run, *known); err != nil { return false, false, begin(r), err } } case *declaration.Container: if known.Recorded(string(declaration.TypeContainer), res.Name) { break } _, exists, err := inspectFound(ctx, res.Name, run) if err != nil { return false, false, begin(r), err } if exists { if isFound, err = found(ctx, r, run, *known); err != nil { return false, false, begin(r), err } break } // Not there under its name, and still it would share what was found: created, it // would mount the predecessor's data beside the predecessor's own container. for _, v := range res.Volumes { src := mountSource(v) key := "volume:" + src if strings.HasPrefix(src, "/") { key = "path:" + src } if src == "" { continue } if trouble := before.why(key); trouble != "" { return false, false, begin(r), fmt.Errorf( "%s, so it is not safe to create a container that would mount it", trouble) } if before.has(key) { isFound, why = true, "would mount "+src+", found on the machine" break } } case *declaration.Directory: isFound = before.has("path:" + res.Path) case *declaration.Archive: isFound = before.has("path:" + res.Path) case *declaration.Process: isFound = before.has("unit-file:" + res.Name) case *declaration.User: isFound = before.has("user:" + res.Name) case *declaration.Service: isFound = before.has("unit:" + res.Unit) } } if !isFound { return false, false, out, nil } out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now) if err != nil { return true, false, out, err } // A held service is not started, stopped, enabled or restarted — but a reload stops nothing, // so one the module names still happens (novox/hq ADR 0102, ADR 0103). if svc, ok := r.(*declaration.Service); ok && svc.State == "running" { if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 { if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" { reloader, can := sys.(serviceReloader) if !can { return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+ "service manager cannot reload a unit", svc.Unit, strings.Join(which, ", ")) } if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil { return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err) } out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing" } } } known.RecordHeld(h) return true, !already || h.Changed != was.Changed, out, nil } // found is whether a declared file or container is present on the machine with no record of this // host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by // a host, whatever this store says, so it is never found. func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) { if known.Recorded(string(r.Kind()), r.Target()) { return false, nil } switch res := r.(type) { case *declaration.File: _, err := os.Lstat(res.Path) if errors.Is(err, os.ErrNotExist) { return false, nil } return err == nil, err case *declaration.Container: seen, exists, err := inspectFound(ctx, res.Name, run) if err != nil || !exists { return false, err } return seen.spec == "", nil } return false, nil } type foundContainer struct { id string running bool spec string } // inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and // whether a host made it. // // **`container inspect`, not the bare form.** A name is not unique across object kinds — a // module regularly names a network the same as the container that joins it (`keycloak` names // both, and it is ordinary). The bare form resolves across every kind and returns whichever it // finds, so a container that does not exist yet but a same-named network does answers with the // network's JSON — no `.State` field at all — and the template below fails to execute rather // than failing to find anything. That reads as "the runtime could not say", which this function's // caller correctly refuses to build on (novox/hq ADR 0100) — but there was something to say, a // question of kind, not of ambiguity that should have stopped anything. func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) { cri, err := containerRuntime(ctx, run) if err != nil { return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name) } out, err := run(ctx, cri, "container", "inspect", "--format", "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) if err != nil { if absent(err) { return foundContainer{}, false, nil } // **A runtime that could not answer is not a machine with nothing there.** Read as // absence, a daemon that is down or a permission denied would let the mesh create its own // container over a predecessor's — the one thing an adopted node must never do // (novox/hq ADR 0100). return foundContainer{}, false, fmt.Errorf( "the container runtime could not say whether %s is here, so it is not safe to make one: %w", name, err) } parts := strings.Split(strings.TrimSpace(out), "\t") for len(parts) < 3 { parts = append(parts, "") } spec := strings.TrimSpace(parts[2]) if spec == "" { spec = "" } return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil } // absent is whether a runtime said the thing is not there, rather than failing to answer. Its own // words: docker and podman both say "No such object", "No such container" or "No such volume". func absent(err error) bool { said := strings.ToLower(err.Error()) for _, missing := range []string{"no such object", "no such container", "no such volume", "no such image"} { if strings.Contains(said, missing) { return true } } return false } // hold keeps a found file or container as it is, and reports it — the first time by recording // what was found, every time after by comparing against that. Nothing is reverted, restarted or // created: a held target that disappears stays held and gone until its module is taken. func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held, already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) { out := begin(r) h := was if !already { h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()), Target: r.Target(), Since: now, Why: why} } h.Module = module detail := "found on the machine; kept until " + module + " is taken" var changed string switch res := r.(type) { case *declaration.File: info, err := os.Lstat(res.Path) switch { case errors.Is(err, os.ErrNotExist): if !already { return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path) } changed = "gone" case err != nil: return out, h, err default: content, err := os.ReadFile(res.Path) if err != nil { return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err) } if !already { // The original first, before anything is recorded: a hold with no kept copy // would be a promise the host cannot keep. if keep == nil { return out, h, fmt.Errorf( "%s was found on this adopted node and this host has nowhere to keep its original", res.Path) } kept, err := keep(res.Path, content, info.Mode().Perm()) if err != nil { return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err) } h.Kept = kept h.Digest = digestOf(string(content)) h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) if st, ok := info.Sys().(*syscall.Stat_t); ok { h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) } } else if digestOf(string(content)) != h.Digest { changed = "rewritten" } } case *declaration.Directory: info, err := os.Lstat(res.Path) switch { case errors.Is(err, os.ErrNotExist): if !already { return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) } changed = "gone" case err != nil: return out, h, err case !already: // Its mode and owner as found, which the mesh leaves: a database refuses to start // on a data directory whose mode changed. h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) if st, ok := info.Sys().(*syscall.Stat_t); ok { h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) } } detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken" case *declaration.Archive: if _, err := os.Lstat(res.Path); errors.Is(err, os.ErrNotExist) { if !already { return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) } changed = "gone" } else if err != nil { return out, h, err } detail = "something is already at " + res.Path + "; nothing unpacked over it or re-owned until " + module + " is taken" case *declaration.Process: unit := filepath.Join(unitDir, res.Name+".service") if !present(unit) { if !already { return out, h, fmt.Errorf("%s was found and is gone before it could be held", unit) } changed = "gone" } detail = "its unit " + unit + " was found on the machine; not written over or restarted until " + module + " is taken" case *declaration.User: _, exists, err := system.LookUpUser(ctx, run, res.Name) switch { case err != nil: return out, h, err case !exists && !already: return out, h, fmt.Errorf("the user %s was found and is gone before it could be held", res.Name) case !exists: changed = "gone" } detail = "the user was found on the machine; its shell and groups are kept until " + module + " is taken" case *declaration.Service: state, err := sys.ServiceState(ctx, run, res.Unit) switch { case err != nil && !already: return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err) case err != nil: changed = "gone" case !already: h.Running = state == "running" case h.Running && state != "running": changed = "stopped" } detail = "its unit was found on the machine; its state and whether it starts at boot are " + "kept until " + module + " is taken" case *declaration.Container: if h.Why != "" && h.Container == "" { // Held for what it would mount, never created: there is nothing of it to compare. detail = "not created: it " + h.Why + "; kept until " + module + " is taken" break } seen, exists, err := inspectFound(ctx, res.Name, run) if err != nil { return out, h, err } switch { case !exists && !already: return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name) case !already: h.Container, h.Running = seen.id, seen.running case !exists: changed = "gone" case seen.id != h.Container: changed = "replaced" case h.Running && !seen.running: changed = "stopped" } default: return out, h, fmt.Errorf("a %s cannot be held", r.Kind()) } if changed != h.Changed { h.Changed = changed h.ChangedAt = now if changed == "" { h.ChangedAt = time.Time{} } } out.Action = "held" out.Detail = detail if h.Changed != "" { out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted" } return out, h, nil } // takenDetail is what an outcome says when a module's cutover replaced what was held for it. func takenDetail(h store.Held) string { if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") { return "taken: no longer held (" + h.Why + ")" } if h.Kept != "" { return "taken: replaced what was found; original kept at " + h.Kept } return "taken: replaced what was found" }