package apply import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "os" "path/filepath" "strings" "syscall" "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // Keep records the original of a file found on an adopted node, before anything else happens to // it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the // first copy is the one that was there before the mesh. type Keep func(path string, content []byte, mode os.FileMode) (string, error) // KeepIn keeps originals under dir/kept, each named for the path it came from, readable by root // alone — a predecessor's configuration may carry its credentials. func KeepIn(dir string) Keep { return func(path string, content []byte, _ os.FileMode) (string, error) { sum := sha256.Sum256([]byte(path)) kept := filepath.Join(dir, "kept", hex.EncodeToString(sum[:])[:16]+"-"+filepath.Base(path)) if _, err := os.Lstat(kept); err == nil { return kept, nil } if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { return "", err } if err := writeAtomically(kept, content, 0o600); err != nil { return "", err } back, err := os.ReadFile(kept) if err != nil || string(back) != string(content) { return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept) } return kept, nil } } // foundBefore is what an adopted apply finds on the machine before it changes anything: each // directory, service unit and container mount source of an untaken module that is present with no // record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a // file's parent directory, a unit file a module writes, a package that brings its unit — and what // the mesh made in this apply was not found. type foundBefore map[string]bool func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State, run Runner) foundBefore { seen := foundBefore{} if d.Adoption == nil { return seen } cri, asked := "", false for _, r := range d.Resources { if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken { continue } if _, held := known.HeldAt(r.Identity()); held { continue } switch res := r.(type) { case *declaration.Directory: if present(res.Path) && !recordedPath(known, res.Path) { seen["path:"+res.Path] = true } case *declaration.Service: if known.Recorded(string(declaration.TypeService), res.Unit) { continue } // A unit the service manager cannot find is not there; one it can read is, whatever // state it is in. if _, err := sys.ServiceState(ctx, run, res.Unit); err == nil { seen["unit:"+res.Unit] = true } case *declaration.Container: if known.Recorded(string(declaration.TypeContainer), res.Name) { continue } for _, v := range res.Volumes { src := mountSource(v) switch { case src == "": case strings.HasPrefix(src, "/"): if present(src) && !recordedPath(known, src) { seen["path:"+src] = true } default: if !asked { cri, _ = containerRuntime(ctx, run) asked = true } if cri == "" { continue } if _, err := run(ctx, cri, "volume", "inspect", src); err == nil { seen["volume:"+src] = true } } } } } return seen } func present(path string) bool { _, err := os.Lstat(path) return err == nil } // recordedPath is whether this host has a record of making something at a path. func recordedPath(known store.State, path string) bool { for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive, declaration.TypeAccess} { if known.Recorded(string(kind), path) { return true } } return false } // mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for // an anonymous volume, which mounts nothing that could already be there. func mountSource(mapping string) string { src, _, ok := strings.Cut(mapping, ":") if !ok { return "" } return src } // runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step // sharing a container's namespace. func runsIn(r declaration.Resource) string { switch res := r.(type) { case *declaration.Action: return res.In case *declaration.Container: if res.RunOnce { if name, ok := strings.CutPrefix(res.Network, "container:"); ok { return name } } } return "" } // heldContainer is what is held under a container's name. func heldContainer(known store.State, name string) (store.Held, bool) { for _, h := range known.Held { if h.Kind == string(declaration.TypeContainer) && h.Target == name { return h, true } } return store.Held{}, false } // holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and // holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no // record is kept as it is: a file or a container under its name, a directory, a service's unit, // and a container that would mount a path or a volume found there. An action or a run-once step // run inside a held container is held with it. Once held, a resource stays held — changed or gone // — until its module is taken, and it is never recorded as applied, so never removed as an orphan. // // Held is false for a resource to apply as usual. News is whether the hold is new or changed, // which is what is worth a line in the log. func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration, known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool, now time.Time) (held, news bool, out Outcome, err error) { was, already := known.HeldAt(r.Identity()) if in := runsIn(r); in != "" { if container, isHeld := heldContainer(*known, in); isHeld { module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) if !untaken { module = container.Module } h := was if !already { h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now} } h.Module, h.Why = module, "runs in "+in known.RecordHeld(h) out = begin(r) out.Action = "held" out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module) return true, !already, out, nil } } module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) if !untaken { return false, false, out, nil } why := was.Why isFound := already if !already { switch res := r.(type) { case *declaration.File: if res.Into == "" { if isFound, err = found(ctx, r, run, *known); err != nil { return false, false, begin(r), err } } case *declaration.Container: if known.Recorded(string(declaration.TypeContainer), res.Name) { break } _, exists, err := inspectFound(ctx, res.Name, run) if err != nil { return false, false, begin(r), err } if exists { if isFound, err = found(ctx, r, run, *known); err != nil { return false, false, begin(r), err } break } // Not there under its name, and still it would share what was found: created, it // would mount the predecessor's data beside the predecessor's own container. for _, v := range res.Volumes { src := mountSource(v) key := "volume:" + src if strings.HasPrefix(src, "/") { key = "path:" + src } if src != "" && before[key] { isFound, why = true, "would mount "+src+", found on the machine" break } } case *declaration.Directory: isFound = before["path:"+res.Path] case *declaration.Service: isFound = before["unit:"+res.Unit] } } if !isFound { return false, false, out, nil } out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now) if err != nil { return true, false, out, err } // A held service is not started, stopped, enabled or restarted — but a reload stops nothing, // so one the module names still happens (novox/hq ADR 0102, ADR 0103). if svc, ok := r.(*declaration.Service); ok && svc.State == "running" { if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 { if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" { reloader, can := sys.(serviceReloader) if !can { return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+ "service manager cannot reload a unit", svc.Unit, strings.Join(which, ", ")) } if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil { return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err) } out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing" } } } known.RecordHeld(h) return true, !already || h.Changed != was.Changed, out, nil } // found is whether a declared file or container is present on the machine with no record of this // host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by // a host, whatever this store says, so it is never found. func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) { if known.Recorded(string(r.Kind()), r.Target()) { return false, nil } switch res := r.(type) { case *declaration.File: _, err := os.Lstat(res.Path) if errors.Is(err, os.ErrNotExist) { return false, nil } return err == nil, err case *declaration.Container: seen, exists, err := inspectFound(ctx, res.Name, run) if err != nil || !exists { return false, err } return seen.spec == "", nil } return false, nil } type foundContainer struct { id string running bool spec string } // inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and // whether a host made it. func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) { cri, err := containerRuntime(ctx, run) if err != nil { return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name) } out, err := run(ctx, cri, "inspect", "--format", "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) if err != nil { return foundContainer{}, false, nil } parts := strings.Split(strings.TrimSpace(out), "\t") for len(parts) < 3 { parts = append(parts, "") } spec := strings.TrimSpace(parts[2]) if spec == "" { spec = "" } return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil } // hold keeps a found file or container as it is, and reports it — the first time by recording // what was found, every time after by comparing against that. Nothing is reverted, restarted or // created: a held target that disappears stays held and gone until its module is taken. func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held, already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) { out := begin(r) h := was if !already { h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()), Target: r.Target(), Since: now, Why: why} } h.Module = module detail := "found on the machine; kept until " + module + " is taken" var changed string switch res := r.(type) { case *declaration.File: info, err := os.Lstat(res.Path) switch { case errors.Is(err, os.ErrNotExist): if !already { return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path) } changed = "gone" case err != nil: return out, h, err default: content, err := os.ReadFile(res.Path) if err != nil { return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err) } if !already { // The original first, before anything is recorded: a hold with no kept copy // would be a promise the host cannot keep. if keep == nil { return out, h, fmt.Errorf( "%s was found on this adopted node and this host has nowhere to keep its original", res.Path) } kept, err := keep(res.Path, content, info.Mode().Perm()) if err != nil { return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err) } h.Kept = kept h.Digest = digestOf(string(content)) h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) if st, ok := info.Sys().(*syscall.Stat_t); ok { h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) } } else if digestOf(string(content)) != h.Digest { changed = "rewritten" } } case *declaration.Directory: info, err := os.Lstat(res.Path) switch { case errors.Is(err, os.ErrNotExist): if !already { return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) } changed = "gone" case err != nil: return out, h, err case !already: // Its mode and owner as found, which the mesh leaves: a database refuses to start // on a data directory whose mode changed. h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) if st, ok := info.Sys().(*syscall.Stat_t); ok { h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) } } detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken" case *declaration.Service: state, err := sys.ServiceState(ctx, run, res.Unit) switch { case err != nil && !already: return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err) case err != nil: changed = "gone" case !already: h.Running = state == "running" case h.Running && state != "running": changed = "stopped" } detail = "its unit was found on the machine; its state and whether it starts at boot are " + "kept until " + module + " is taken" case *declaration.Container: if h.Why != "" && h.Container == "" { // Held for what it would mount, never created: there is nothing of it to compare. detail = "not created: it " + h.Why + "; kept until " + module + " is taken" break } seen, exists, err := inspectFound(ctx, res.Name, run) if err != nil { return out, h, err } switch { case !exists && !already: return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name) case !already: h.Container, h.Running = seen.id, seen.running case !exists: changed = "gone" case seen.id != h.Container: changed = "replaced" case h.Running && !seen.running: changed = "stopped" } default: return out, h, fmt.Errorf("a %s cannot be held", r.Kind()) } if changed != h.Changed { h.Changed = changed h.ChangedAt = now if changed == "" { h.ChangedAt = time.Time{} } } out.Action = "held" out.Detail = detail if h.Changed != "" { out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted" } return out, h, nil } // takenDetail is what an outcome says when a module's cutover replaced what was held for it. func takenDetail(h store.Held) string { if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") { return "taken: no longer held (" + h.Why + ")" } if h.Kept != "" { return "taken: replaced what was found; original kept at " + h.Kept } return "taken: replaced what was found" }