Files
mesh-host/internal/upgrade/witnessed.go
T
jochen 0c405b70cc Roll a core build back by a witness that is not the new build (hq to-be 45 Phase 4)
The launcher trusted a counter only a by-hand reconcile ever cleared and a
known-good nothing in the daemon wrote, so no machine could roll its host back;
the controller and the node tools were replaced in place with nothing kept.

- The launcher runs a delivered host that is not known-good on trial: one that
  crashes, stops for nothing, or does not report within ten minutes goes back
  to known-good, once per version, recorded in rolled-back. The host proves
  itself when the mesh takes a report under its own build, says every standing
  verdict on its reports, never stands aside for a rolled-back version, and
  restarts its service once when its launcher was replaced on disk.
- The engine keeps the controller's and the node tools' previous build beside
  the new one and judges the new one: the lease taken by the controller it
  started (read-only direct get of mesh-controller_lease/holder), or this
  machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds
  of time it could ask. Not healthy: the previous restored, once, said. Proved:
  the previous deleted. A build declared not-reversible is never rolled back.
- Retire never removes a version newer than the running one.
2026-10-06 18:23:56 +02:00

132 lines
4.4 KiB
Go

package upgrade
import (
"bufio"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
// The host's own successor, witnessed by its launcher (novox/hq to-be 45 §8, ADR 0227 rule 8).
//
// The launcher runs a delivered host that is not the known-good one **on trial**, and rolls back
// from one that crashes repeatedly, stops for nothing, or does not report within its bound. Two
// files are the whole conversation between them, both plain enough for a shell:
//
// - known-good — written by this host when the mesh has taken a report it made about its
// declaration under its own build. That is what ends a trial.
// - rolled-back — written by the launcher: one line per verdict, tab-separated — from, to, when
// (seconds since the epoch), outcome, why. Read here and said on every report while it stands.
// RolledBackName is the launcher's record of its verdicts, beside the state.
const RolledBackName = "rolled-back"
// RolledBackPath is where it lives, given where the state lives.
func RolledBackPath(statePath string) string {
return filepath.Join(filepath.Dir(statePath), RolledBackName)
}
// Verdict is one line of the launcher's record.
type Verdict struct {
From, To, Outcome, Why string
At time.Time
}
// ReadRolledBack is the launcher's record, oldest first. A line it cannot read is skipped and named
// in the error, never guessed at; absence is no verdicts.
func ReadRolledBack(path string) ([]Verdict, error) {
file, err := os.Open(path)
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
defer file.Close()
var out []Verdict
var bad []string
lines := bufio.NewScanner(file)
for n := 1; lines.Scan(); n++ {
line := strings.TrimRight(lines.Text(), "\r")
if strings.TrimSpace(line) == "" {
continue
}
fields := strings.SplitN(line, "\t", 5)
if len(fields) != 5 || fields[0] == "" {
bad = append(bad, strconv.Itoa(n))
continue
}
seconds, err := strconv.ParseInt(fields[2], 10, 64)
if err != nil {
bad = append(bad, strconv.Itoa(n))
continue
}
out = append(out, Verdict{From: fields[0], To: fields[1], At: time.Unix(seconds, 0).UTC(),
Outcome: fields[3], Why: fields[4]})
}
if err := lines.Err(); err != nil {
return out, err
}
if len(bad) > 0 {
return out, fmt.Errorf("the launcher's record %s has line(s) %s that are not from, to, when, outcome, why",
path, strings.Join(bad, ", "))
}
return out, nil
}
// RolledBackVersions are the versions the launcher refuses to start again.
func RolledBackVersions(verdicts []Verdict) []string {
var out []string
for _, v := range verdicts {
out = append(out, v.From)
}
return out
}
// Proved is this host's version seen to report its declaration under its own build: the evidence a
// trial waits for, and the only evidence known-good has ever claimed (novox/hq ADR 0005).
//
// - known-good becomes this version, which ends the launcher's trial;
// - the start counter is cleared, so months of ordinary restarts never add up to a rollback;
// - versions older than this one's predecessor are retired — never one newer, never this one;
// - and when this version is not the one a rollback went back to, the launcher's verdicts end: a
// newer host has proved itself, and what was concluded about an older one no longer stands.
//
// Returns the versions retired. Every step is attempted; the errors are joined.
func Proved(statePath, versionsDir, version string) ([]string, error) {
if strings.TrimSpace(version) == "" {
return nil, errors.New("a host that does not know its own version cannot prove it")
}
var errs []error
if err := RecordKnownGood(KnownGoodPath(statePath), version); err != nil {
errs = append(errs, fmt.Errorf("recording %s as known-good: %w", version, err))
}
if err := ClearAttempts(AttemptsPath(statePath)); err != nil {
errs = append(errs, fmt.Errorf("clearing the start counter: %w", err))
}
retired, err := Retire(versionsDir, version)
if err != nil {
errs = append(errs, err)
}
verdicts, err := ReadRolledBack(RolledBackPath(statePath))
if err != nil {
errs = append(errs, err)
}
wentBackTo := false
for _, v := range verdicts {
if v.To == version {
wentBackTo = true
}
}
if len(verdicts) > 0 && !wentBackTo {
if err := os.Remove(RolledBackPath(statePath)); err != nil && !errors.Is(err, os.ErrNotExist) {
errs = append(errs, err)
}
}
return retired, errors.Join(errs...)
}