package witness import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "sort" "time" "github.com/novox/mesh-host/internal/link" ) // What the engine keeps beside a witnessed process (to-be 45 §8): the build before the running one, // and what it knows about the running one — on trial or proved, and what was concluded. // // / the running build, where its unit runs it from // /.witness//previous/ the build before it, kept until the running one is proved // /.witness//state.json State // // **Never deleted before the new build is proved**, and retired once it is: the previous build has // exactly one reader — a restoration — and none once the build after it has been seen healthy. Retired, // not deleted: a build is deleted only once no process runs from it (draining.go, issue 289). // The running build's directory never moves while it is judged, so its unit is the same unit // throughout, and restoring is two renames and a restart. // Dir is where the engine keeps what it knows about a witnessed process. func Dir(root, name string) string { return filepath.Join(root, ".witness", name) } func previousDir(root, name string) string { return filepath.Join(Dir(root, name), "previous") } func statePath(root, name string) string { return filepath.Join(Dir(root, name), "state.json") } // State is one witnessed process, as the engine keeps it. type State struct { Process string `json:"process"` Witness string `json:"witness"` // Running is the digest of the build at /; Proven, whether it has been seen healthy // (or ran before any witness watched it, or is a build restored — judged once already). Running string `json:"running"` Proven bool `json:"proven"` // Previous is the digest of the build kept to go back to, while Running is on trial. Previous string `json:"previous,omitempty"` // Started is when Running was placed; Watched how long the witness has judged it while it could // ask, Unasked how long it could not. Within is its bound. Started time.Time `json:"started,omitempty"` Watched time.Duration `json:"watched,omitempty"` Unasked time.Duration `json:"unasked,omitempty"` Within time.Duration `json:"within,omitempty"` // NotReversible is why Running may not be rolled back, as its declaration said: the build // before it would run against what this one changed. NotReversible string `json:"not-reversible,omitempty"` // Verdicts are what the witness concluded and still stands: said on every report until the mesh // asks for another build, or a build is proved. Verdicts []link.Rollback `json:"verdicts,omitempty"` // Refused are builds rolled back here: one rollback per build, so a build in this list is not // placed again until a newer one is proved. Refused []string `json:"refused,omitempty"` } // OnTrial says whether the running build is still being judged. func (s State) OnTrial() bool { if s.Proven || s.Running == "" { return false } for _, v := range s.Verdicts { if v.From == s.Running { return false } } return true } func (s State) refuses(digest string) bool { for _, d := range s.Refused { if d == digest { return true } } return false } // Load is what the engine keeps about one process; a zero State when it keeps nothing. func Load(root, name string) (State, error) { raw, err := os.ReadFile(statePath(root, name)) if errors.Is(err, os.ErrNotExist) { return State{}, nil } if err != nil { return State{}, err } var s State if err := json.Unmarshal(raw, &s); err != nil { return State{}, fmt.Errorf("what the engine keeps about %s cannot be read: %w", name, err) } return s, nil } // Save keeps it, whole or not at all. func Save(root string, s State) error { dir := Dir(root, s.Process) if err := keep(root, s.Process); err != nil { return err } body, err := json.MarshalIndent(s, "", " ") if err != nil { return err } tmp, err := os.CreateTemp(dir, ".state-*") if err != nil { return err } defer os.Remove(tmp.Name()) if _, err := tmp.Write(body); err != nil { tmp.Close() return err } if err := tmp.Sync(); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } return os.Rename(tmp.Name(), statePath(root, s.Process)) } // Forget is a witnessed process no longer declared: everything kept about it goes with it. func Forget(root, name string) error { return os.RemoveAll(Dir(root, name)) } // Placing is what the applier is to do with a declared build of a witnessed process. type Placing struct { // Unpack is whether the declared build is to be unpacked at /; false when the build // there already is the one to run. Unpack bool // Detail is what to say about it, when anything. Detail string // Commit records the placement once the build is unpacked and started. Commit func() error } // Place readies / for a declared build of a witnessed process, before anything is // unpacked there. `recorded` is the digest the host's record says it placed last, for a process the // engine keeps nothing about yet — every one on the day this ships. // // - the declared build is the one running (restored here, or declared again): nothing is unpacked. // - it was rolled back here and nothing newer has been proved: refused, and the running build stays. // - the running build is proved: it is moved aside as the previous one, and the new one goes on trial. // - the running build is itself on trial: it is discarded, and the previous one stays the one to go // back to — the last build seen healthy, never one that was not. // - nothing runs there yet: a first placement, with nothing to go back to and nothing to judge. func Place(root, name, by, declared, recorded, notReversible string, now time.Time) (Placing, error) { at := filepath.Join(root, name) s, err := Load(root, name) if err != nil { return Placing{}, err } if s.Process == "" { // Nothing kept: what is there is whatever the record says, and it ran before any witness — // it is the build a trial would go back to. s = State{Process: name, Running: recorded, Proven: true} } s.Witness = by present := false if info, err := os.Stat(at); err == nil && info.IsDir() { present = true } if present && s.Running == declared { // Asked for the build already running. A restoration followed by the mesh asking for that // same build again ends what was concluded about the build it replaced: the mesh asks for what // runs. What was concluded about this build itself stands. var standing []link.Rollback for _, v := range s.Verdicts { if v.From == s.Running { standing = append(standing, v) } } s.Verdicts = standing return Placing{Commit: func() error { return Save(root, s) }}, nil } if present && s.refuses(declared) { return Placing{ Detail: fmt.Sprintf("kept build %s: %s was rolled back on this machine and is not placed again "+ "until a newer build has proved itself (novox/hq to-be 45 §8)", short(s.Running), short(declared)), Commit: func() error { return Save(root, s) }, }, nil } previous := s.Previous switch { case !present || s.Running == "": // A first placement, or a directory that went missing: nothing to keep. if err := os.RemoveAll(at); err != nil { return Placing{}, err } s = State{Process: name, Witness: by, Running: declared, Proven: true, Refused: s.Refused} return Placing{Unpack: true, Commit: func() error { return Save(root, s) }}, nil case s.OnTrial(): // The running build has not been seen healthy: it is not what anybody goes back to. Retired, // since it still runs until the restart (issue 289). if err := retire(root, name, at, now); err != nil { return Placing{}, err } default: if err := retire(root, name, previousDir(root, name), now); err != nil { return Placing{}, err } // Moved while its process still runs, until the restart stops it: reachable at its new path // by the user it runs as (issue 289). if err := keep(root, name); err != nil { return Placing{}, err } if err := os.Rename(at, previousDir(root, name)); err != nil { return Placing{}, fmt.Errorf("cannot keep %s's running build to go back to: %w", name, err) } previous = s.Running } s = State{Process: name, Witness: by, Running: declared, Previous: previous, Started: now.UTC(), Within: Within(by), NotReversible: notReversible, Refused: s.Refused} // Kept as soon as the old build is aside, so a host that stops here knows on its return which // build is where. if err := Save(root, s); err != nil { return Placing{}, err } return Placing{Unpack: true, Commit: func() error { return Save(root, s) }}, nil } // Proved is the running build seen healthy: the build before it is retired — it has no reader now, // and is deleted once no process runs from it — and what was concluded and refused before it ends. func Proved(root, name string) error { s, err := Load(root, name) if err != nil || s.Process == "" { return err } if err := retire(root, name, previousDir(root, name), time.Now()); err != nil { return err } s.Proven, s.Previous, s.Verdicts, s.Refused = true, "", nil, nil s.Watched, s.Unasked = 0, 0 return Save(root, s) } // Runner is how the engine asks the machine's service manager, as the applier does. type Runner func(ctx context.Context, name string, args ...string) (string, error) // Restore is the running build not healthy in bound: the previous one put back and started, once — // or, when the running build is declared not reversible or nothing is kept, nothing done and that // said. The verdict is kept, and returned to be said. func Restore(ctx context.Context, root, name, why string, run Runner, now time.Time) (link.Rollback, error) { s, err := Load(root, name) if err != nil { return link.Rollback{}, err } v := link.Rollback{Component: Component(s.Witness), From: s.Running, Why: why, At: now.UTC()} conclude := func(v link.Rollback) (link.Rollback, error) { s.Verdicts = append(s.Verdicts, v) return v, Save(root, s) } switch { case s.NotReversible != "": v.Outcome = link.NotReversible v.Why = why + "; not rolled back: this build is declared not reversible (" + s.NotReversible + "), so the build before it would run against what it changed. It is left running. A person decides" return conclude(v) case s.Previous == "": v.Outcome = link.NothingToRestore v.Why = why + "; no build before it is kept on this machine" return conclude(v) } if _, err := os.Stat(previousDir(root, name)); err != nil { v.Outcome = link.NothingToRestore v.Why = fmt.Sprintf("%s; the build before it (%s) is not where it was kept: %v", why, short(s.Previous), err) return conclude(v) } unit := name + ".service" // Stopped first, so the failing build is not running while its files are moved; a stop that fails // is not fatal — the restart below replaces whatever runs. _, _ = run(ctx, "systemctl", "stop", unit) at := filepath.Join(root, name) failed := filepath.Join(Dir(root, name), "failed") if err := retire(root, name, failed, now); err != nil { return restoreFailed(root, s, v, err) } if err := os.Rename(at, failed); err != nil && !errors.Is(err, os.ErrNotExist) { return restoreFailed(root, s, v, err) } if err := os.Rename(previousDir(root, name), at); err != nil { // Put back what was there, so the machine is no worse than before the attempt. _ = os.Rename(failed, at) return restoreFailed(root, s, v, err) } _ = retire(root, name, failed, now) v.To, v.Outcome = s.Previous, link.RolledBack s.Refused = append(s.Refused, s.Running) // The restored build was proved before; it is not judged a second time. One rollback per build. s.Running, s.Previous, s.Proven = s.Previous, "", true if _, err := run(ctx, "systemctl", "restart", unit); err != nil { v.Outcome = link.RestoreFailed v.Why = fmt.Sprintf("%s; the build before it (%s) was put back and would not start: %v", why, short(v.To), err) } return conclude(v) } func restoreFailed(root string, s State, v link.Rollback, err error) (link.Rollback, error) { v.Outcome = link.RestoreFailed v.Why = fmt.Sprintf("%s; putting the build before it (%s) back failed: %v", v.Why, short(s.Previous), err) s.Verdicts = append(s.Verdicts, v) return v, Save(root, s) } // Conclude keeps a verdict that restores nothing — a build that could not be judged at all. func Conclude(root, name string, v link.Rollback) error { s, err := Load(root, name) if err != nil { return err } s.Verdicts = append(s.Verdicts, v) return Save(root, s) } // Standing is every verdict that still stands, on every witnessed process under root, in a stable // order — what every report says. func Standing(root string) []link.Rollback { var out []link.Rollback for _, s := range all(root) { out = append(out, s.Verdicts...) } sort.SliceStable(out, func(i, j int) bool { if out[i].Component != out[j].Component { return out[i].Component < out[j].Component } return out[i].At.Before(out[j].At) }) return out } // Trials is every witnessed process whose running build is being judged. func Trials(root string) []State { var out []State for _, s := range all(root) { if s.OnTrial() { out = append(out, s) } } return out } func all(root string) []State { entries, err := os.ReadDir(filepath.Join(root, ".witness")) if err != nil { return nil } var out []State for _, e := range entries { if !e.IsDir() { continue } if s, err := Load(root, e.Name()); err == nil && s.Process != "" { out = append(out, s) } } sort.Slice(out, func(i, j int) bool { return out[i].Process < out[j].Process }) return out } func short(digest string) string { if len(digest) > len("sha256:")+12 { return digest[:len("sha256:")+12] } return digest }