The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
132 lines
4.4 KiB
Go
132 lines
4.4 KiB
Go
package upgrade
|
|
|
|
import (
|
|
"bufio"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// The host's own successor, witnessed by its launcher (novox/hq to-be 45 §8, ADR 0227 rule 8).
|
|
//
|
|
// The launcher runs a delivered host that is not the known-good one **on trial**, and rolls back
|
|
// from one that crashes repeatedly, stops for nothing, or does not report within its bound. Two
|
|
// files are the whole conversation between them, both plain enough for a shell:
|
|
//
|
|
// - known-good — written by this host when the mesh has taken a report it made about its
|
|
// declaration under its own build. That is what ends a trial.
|
|
// - rolled-back — written by the launcher: one line per verdict, tab-separated — from, to, when
|
|
// (seconds since the epoch), outcome, why. Read here and said on every report while it stands.
|
|
|
|
// RolledBackName is the launcher's record of its verdicts, beside the state.
|
|
const RolledBackName = "rolled-back"
|
|
|
|
// RolledBackPath is where it lives, given where the state lives.
|
|
func RolledBackPath(statePath string) string {
|
|
return filepath.Join(filepath.Dir(statePath), RolledBackName)
|
|
}
|
|
|
|
// Verdict is one line of the launcher's record.
|
|
type Verdict struct {
|
|
From, To, Outcome, Why string
|
|
At time.Time
|
|
}
|
|
|
|
// ReadRolledBack is the launcher's record, oldest first. A line it cannot read is skipped and named
|
|
// in the error, never guessed at; absence is no verdicts.
|
|
func ReadRolledBack(path string) ([]Verdict, error) {
|
|
file, err := os.Open(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
var out []Verdict
|
|
var bad []string
|
|
lines := bufio.NewScanner(file)
|
|
for n := 1; lines.Scan(); n++ {
|
|
line := strings.TrimRight(lines.Text(), "\r")
|
|
if strings.TrimSpace(line) == "" {
|
|
continue
|
|
}
|
|
fields := strings.SplitN(line, "\t", 5)
|
|
if len(fields) != 5 || fields[0] == "" {
|
|
bad = append(bad, strconv.Itoa(n))
|
|
continue
|
|
}
|
|
seconds, err := strconv.ParseInt(fields[2], 10, 64)
|
|
if err != nil {
|
|
bad = append(bad, strconv.Itoa(n))
|
|
continue
|
|
}
|
|
out = append(out, Verdict{From: fields[0], To: fields[1], At: time.Unix(seconds, 0).UTC(),
|
|
Outcome: fields[3], Why: fields[4]})
|
|
}
|
|
if err := lines.Err(); err != nil {
|
|
return out, err
|
|
}
|
|
if len(bad) > 0 {
|
|
return out, fmt.Errorf("the launcher's record %s has line(s) %s that are not from, to, when, outcome, why",
|
|
path, strings.Join(bad, ", "))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// RolledBackVersions are the versions the launcher refuses to start again.
|
|
func RolledBackVersions(verdicts []Verdict) []string {
|
|
var out []string
|
|
for _, v := range verdicts {
|
|
out = append(out, v.From)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Proved is this host's version seen to report its declaration under its own build: the evidence a
|
|
// trial waits for, and the only evidence known-good has ever claimed (novox/hq ADR 0005).
|
|
//
|
|
// - known-good becomes this version, which ends the launcher's trial;
|
|
// - the start counter is cleared, so months of ordinary restarts never add up to a rollback;
|
|
// - versions older than this one's predecessor are retired — never one newer, never this one;
|
|
// - and when this version is not the one a rollback went back to, the launcher's verdicts end: a
|
|
// newer host has proved itself, and what was concluded about an older one no longer stands.
|
|
//
|
|
// Returns the versions retired. Every step is attempted; the errors are joined.
|
|
func Proved(statePath, versionsDir, version string) ([]string, error) {
|
|
if strings.TrimSpace(version) == "" {
|
|
return nil, errors.New("a host that does not know its own version cannot prove it")
|
|
}
|
|
var errs []error
|
|
if err := RecordKnownGood(KnownGoodPath(statePath), version); err != nil {
|
|
errs = append(errs, fmt.Errorf("recording %s as known-good: %w", version, err))
|
|
}
|
|
if err := ClearAttempts(AttemptsPath(statePath)); err != nil {
|
|
errs = append(errs, fmt.Errorf("clearing the start counter: %w", err))
|
|
}
|
|
retired, err := Retire(versionsDir, version)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
verdicts, err := ReadRolledBack(RolledBackPath(statePath))
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
wentBackTo := false
|
|
for _, v := range verdicts {
|
|
if v.To == version {
|
|
wentBackTo = true
|
|
}
|
|
}
|
|
if len(verdicts) > 0 && !wentBackTo {
|
|
if err := os.Remove(RolledBackPath(statePath)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
return retired, errors.Join(errs...)
|
|
}
|