mesh-host/internal/witness/kept.go

package witness

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"time"

	"github.com/novox/mesh-host/internal/link"
)

// What the engine keeps beside a witnessed process (to-be 45 §8): the build before the running one,
// and what it knows about the running one — on trial or proved, and what was concluded.
//
//	<root>/<name>                         the running build, where its unit runs it from
//	<root>/.witness/<name>/previous/      the build before it, kept until the running one is proved
//	<root>/.witness/<name>/state.json     State
//
// **Never deleted before the new build is proved**, and retired once it is: the previous build has
// exactly one reader — a restoration — and none once the build after it has been seen healthy. Retired,
// not deleted: a build is deleted only once no process runs from it (draining.go, issue 289).
// The running build's directory never moves while it is judged, so its unit is the same unit
// throughout, and restoring is two renames and a restart.

// Dir is where the engine keeps what it knows about a witnessed process.
func Dir(root, name string) string { return filepath.Join(root, ".witness", name) }

func previousDir(root, name string) string { return filepath.Join(Dir(root, name), "previous") }
func statePath(root, name string) string   { return filepath.Join(Dir(root, name), "state.json") }

// State is one witnessed process, as the engine keeps it.
type State struct {
	Process string `json:"process"`
	Witness string `json:"witness"`
	// Running is the digest of the build at <root>/<name>; Proven, whether it has been seen healthy
	// (or ran before any witness watched it, or is a build restored — judged once already).
	Running string `json:"running"`
	Proven  bool   `json:"proven"`
	// Previous is the digest of the build kept to go back to, while Running is on trial.
	Previous string `json:"previous,omitempty"`
	// Started is when Running was placed; Watched how long the witness has judged it while it could
	// ask, Unasked how long it could not. Within is its bound.
	Started time.Time     `json:"started,omitempty"`
	Watched time.Duration `json:"watched,omitempty"`
	Unasked time.Duration `json:"unasked,omitempty"`
	Within  time.Duration `json:"within,omitempty"`
	// NotReversible is why Running may not be rolled back, as its declaration said: the build
	// before it would run against what this one changed.
	NotReversible string `json:"not-reversible,omitempty"`
	// Verdicts are what the witness concluded and still stands: said on every report until the mesh
	// asks for another build, or a build is proved.
	Verdicts []link.Rollback `json:"verdicts,omitempty"`
	// Refused are builds rolled back here: one rollback per build, so a build in this list is not
	// placed again until a newer one is proved.
	Refused []string `json:"refused,omitempty"`
}

// OnTrial says whether the running build is still being judged.
func (s State) OnTrial() bool {
	if s.Proven || s.Running == "" {
		return false
	}
	for _, v := range s.Verdicts {
		if v.From == s.Running {
			return false
		}
	}
	return true
}

func (s State) refuses(digest string) bool {
	for _, d := range s.Refused {
		if d == digest {
			return true
		}
	}
	return false
}

// Load is what the engine keeps about one process; a zero State when it keeps nothing.
func Load(root, name string) (State, error) {
	raw, err := os.ReadFile(statePath(root, name))
	if errors.Is(err, os.ErrNotExist) {
		return State{}, nil
	}
	if err != nil {
		return State{}, err
	}
	var s State
	if err := json.Unmarshal(raw, &s); err != nil {
		return State{}, fmt.Errorf("what the engine keeps about %s cannot be read: %w", name, err)
	}
	return s, nil
}

// Save keeps it, whole or not at all.
func Save(root string, s State) error {
	dir := Dir(root, s.Process)
	if err := keep(root, s.Process); err != nil {
		return err
	}
	body, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return err
	}
	tmp, err := os.CreateTemp(dir, ".state-*")
	if err != nil {
		return err
	}
	defer os.Remove(tmp.Name())
	if _, err := tmp.Write(body); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	return os.Rename(tmp.Name(), statePath(root, s.Process))
}

// Forget is a process no longer witnessed, or no longer declared: everything kept about it goes with it,
// but for a build a process still runs from. That one is retired and deleted once nothing runs from it
// (draining.go): a process whose witness was taken away, or which never had one, is still running from
// whatever was retired for it until its unit restarts (novox/hq issue 734).
func Forget(root, name string) error {
	dir := Dir(root, name)
	if _, err := os.Lstat(dir); os.IsNotExist(err) {
		return nil
	}
	if err := retire(root, name, previousDir(root, name), time.Now()); err != nil {
		return err
	}
	if err := os.Remove(statePath(root, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
		return err
	}
	if draining, err := Sweep(root, name); err != nil || len(draining) > 0 {
		return err
	}
	return os.RemoveAll(dir)
}

// Placing is what the applier is to do with a declared build of a witnessed process.
type Placing struct {
	// Unpack is whether the declared build is to be unpacked at <root>/<name>; false when the build
	// there already is the one to run.
	Unpack bool
	// Detail is what to say about it, when anything.
	Detail string
	// Commit records the placement once the build is unpacked and started.
	Commit func() error
}

// Place readies <root>/<name> for a declared build of a witnessed process, before anything is
// unpacked there. `recorded` is the digest the host's record says it placed last, for a process the
// engine keeps nothing about yet — every one on the day this ships.
//
//   - the declared build is the one running (restored here, or declared again): nothing is unpacked.
//   - it was rolled back here and nothing newer has been proved: refused, and the running build stays.
//   - the running build is proved: it is moved aside as the previous one, and the new one goes on trial.
//   - the running build is itself on trial: it is discarded, and the previous one stays the one to go
//     back to — the last build seen healthy, never one that was not.
//   - nothing runs there yet: a first placement, with nothing to go back to and nothing to judge.
func Place(root, name, by, declared, recorded, notReversible string, now time.Time) (Placing, error) {
	at := filepath.Join(root, name)
	s, err := Load(root, name)
	if err != nil {
		return Placing{}, err
	}
	if s.Process == "" {
		// Nothing kept: what is there is whatever the record says, and it ran before any witness —
		// it is the build a trial would go back to.
		s = State{Process: name, Running: recorded, Proven: true}
	}
	s.Witness = by
	present := false
	if info, err := os.Stat(at); err == nil && info.IsDir() {
		present = true
	}

	if present && s.Running == declared {
		// Asked for the build already running. A restoration followed by the mesh asking for that
		// same build again ends what was concluded about the build it replaced: the mesh asks for what
		// runs. What was concluded about this build itself stands.
		var standing []link.Rollback
		for _, v := range s.Verdicts {
			if v.From == s.Running {
				standing = append(standing, v)
			}
		}
		s.Verdicts = standing
		return Placing{Commit: func() error { return Save(root, s) }}, nil
	}
	if present && s.refuses(declared) {
		return Placing{
			Detail: fmt.Sprintf("kept build %s: %s was rolled back on this machine and is not placed again "+
				"until a newer build has proved itself (novox/hq to-be 45 §8)", short(s.Running), short(declared)),
			Commit: func() error { return Save(root, s) },
		}, nil
	}

	previous := s.Previous
	switch {
	case !present || s.Running == "":
		// A first placement, or a directory that went missing: nothing to keep.
		if err := os.RemoveAll(at); err != nil {
			return Placing{}, err
		}
		s = State{Process: name, Witness: by, Running: declared, Proven: true, Refused: s.Refused}
		return Placing{Unpack: true, Commit: func() error { return Save(root, s) }}, nil
	case s.OnTrial():
		// The running build has not been seen healthy: it is not what anybody goes back to. Retired,
		// since it still runs until the restart (issue 289).
		if err := retire(root, name, at, now); err != nil {
			return Placing{}, err
		}
	default:
		if err := retire(root, name, previousDir(root, name), now); err != nil {
			return Placing{}, err
		}
		// Moved while its process still runs, until the restart stops it: reachable at its new path
		// by the user it runs as (issue 289).
		if err := keep(root, name); err != nil {
			return Placing{}, err
		}
		if err := os.Rename(at, previousDir(root, name)); err != nil {
			return Placing{}, fmt.Errorf("cannot keep %s's running build to go back to: %w", name, err)
		}
		previous = s.Running
	}
	s = State{Process: name, Witness: by, Running: declared, Previous: previous, Started: now.UTC(),
		Within: Within(by), NotReversible: notReversible, Refused: s.Refused}
	// Kept as soon as the old build is aside, so a host that stops here knows on its return which
	// build is where.
	if err := Save(root, s); err != nil {
		return Placing{}, err
	}
	return Placing{Unpack: true, Commit: func() error { return Save(root, s) }}, nil
}

// Proved is the running build seen healthy: the build before it is retired — it has no reader now,
// and is deleted once no process runs from it — and what was concluded and refused before it ends.
func Proved(root, name string) error {
	s, err := Load(root, name)
	if err != nil || s.Process == "" {
		return err
	}
	if err := retire(root, name, previousDir(root, name), time.Now()); err != nil {
		return err
	}
	s.Proven, s.Previous, s.Verdicts, s.Refused = true, "", nil, nil
	s.Watched, s.Unasked = 0, 0
	return Save(root, s)
}

// Runner is how the engine asks the machine's service manager, as the applier does.
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// Restore is the running build not healthy in bound: the previous one put back and started, once —
// or, when the running build is declared not reversible or nothing is kept, nothing done and that
// said. The verdict is kept, and returned to be said.
func Restore(ctx context.Context, root, name, why string, run Runner, now time.Time) (link.Rollback, error) {
	s, err := Load(root, name)
	if err != nil {
		return link.Rollback{}, err
	}
	v := link.Rollback{Component: Component(s.Witness), From: s.Running, Why: why, At: now.UTC()}
	conclude := func(v link.Rollback) (link.Rollback, error) {
		s.Verdicts = append(s.Verdicts, v)
		return v, Save(root, s)
	}
	switch {
	case s.NotReversible != "":
		v.Outcome = link.NotReversible
		v.Why = why + "; not rolled back: this build is declared not reversible (" + s.NotReversible +
			"), so the build before it would run against what it changed. It is left running. A person decides"
		return conclude(v)
	case s.Previous == "":
		v.Outcome = link.NothingToRestore
		v.Why = why + "; no build before it is kept on this machine"
		return conclude(v)
	}
	if _, err := os.Stat(previousDir(root, name)); err != nil {
		v.Outcome = link.NothingToRestore
		v.Why = fmt.Sprintf("%s; the build before it (%s) is not where it was kept: %v", why, short(s.Previous), err)
		return conclude(v)
	}

	unit := name + ".service"
	// Stopped first, so the failing build is not running while its files are moved; a stop that fails
	// is not fatal — the restart below replaces whatever runs.
	_, _ = run(ctx, "systemctl", "stop", unit)
	at := filepath.Join(root, name)
	failed := filepath.Join(Dir(root, name), "failed")
	if err := retire(root, name, failed, now); err != nil {
		return restoreFailed(root, s, v, err)
	}
	if err := os.Rename(at, failed); err != nil && !errors.Is(err, os.ErrNotExist) {
		return restoreFailed(root, s, v, err)
	}
	if err := os.Rename(previousDir(root, name), at); err != nil {
		// Put back what was there, so the machine is no worse than before the attempt.
		_ = os.Rename(failed, at)
		return restoreFailed(root, s, v, err)
	}
	_ = retire(root, name, failed, now)

	v.To, v.Outcome = s.Previous, link.RolledBack
	s.Refused = append(s.Refused, s.Running)
	// The restored build was proved before; it is not judged a second time. One rollback per build.
	s.Running, s.Previous, s.Proven = s.Previous, "", true
	if _, err := run(ctx, "systemctl", "restart", unit); err != nil {
		v.Outcome = link.RestoreFailed
		v.Why = fmt.Sprintf("%s; the build before it (%s) was put back and would not start: %v", why, short(v.To), err)
	}
	return conclude(v)
}

func restoreFailed(root string, s State, v link.Rollback, err error) (link.Rollback, error) {
	v.Outcome = link.RestoreFailed
	v.Why = fmt.Sprintf("%s; putting the build before it (%s) back failed: %v", v.Why, short(s.Previous), err)
	s.Verdicts = append(s.Verdicts, v)
	return v, Save(root, s)
}

// Conclude keeps a verdict that restores nothing — a build that could not be judged at all.
func Conclude(root, name string, v link.Rollback) error {
	s, err := Load(root, name)
	if err != nil {
		return err
	}
	s.Verdicts = append(s.Verdicts, v)
	return Save(root, s)
}

// Standing is every verdict that still stands, on every witnessed process under root, in a stable
// order — what every report says.
func Standing(root string) []link.Rollback {
	var out []link.Rollback
	for _, s := range all(root) {
		out = append(out, s.Verdicts...)
	}
	sort.SliceStable(out, func(i, j int) bool {
		if out[i].Component != out[j].Component {
			return out[i].Component < out[j].Component
		}
		return out[i].At.Before(out[j].At)
	})
	return out
}

// Trials is every witnessed process whose running build is being judged.
func Trials(root string) []State {
	var out []State
	for _, s := range all(root) {
		if s.OnTrial() {
			out = append(out, s)
		}
	}
	return out
}

func all(root string) []State {
	entries, err := os.ReadDir(filepath.Join(root, ".witness"))
	if err != nil {
		return nil
	}
	var out []State
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		if s, err := Load(root, e.Name()); err == nil && s.Process != "" {
			out = append(out, s)
		}
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Process < out[j].Process })
	return out
}

func short(digest string) string {
	if len(digest) > len("sha256:")+12 {
		return digest[:len("sha256:")+12]
	}
	return digest
}