novox/mesh-host
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Hand a replaced resource over to the process that replaces it (hq issue 213)

Browse Source 
The controller moves from a container to a process on the one machine
that runs it (novox/hq issue 213). Every orphan is removed before anything
is applied, so the container would go first and nothing would answer the
mesh's verbs while the process was fetched, unpacked and started — and
never again, if it did not start.

- a process may say what it `replaces`: resources the declaration no
  longer declares. Such an orphan is kept through the up-front sweep and
  removed right after the process applied and is up: active and running
  at two looks ten seconds apart, the same main process, no restart in
  between (stricter than ADR 0184's second look, which reads a unit
  waiting to restart as running). If the process failed, was skipped
  behind its module's step, or is not running, the orphan stays running
  and recorded, reported kept, and the next apply hands it over.
  Refused: naming something still declared, itself, an empty id, one
  thing named by two processes, and `replaces` on a step or a schedule.
- a run-once process is run by a oneshot unit, so a step has its
  process's user, working directory and environment and `./name` is its
  own bundle's binary; it was run directly by the host, as root, with no
  environment. It is not enabled.
- a run-once process that fails gates its module, as a run-once
  container already did, so a version whose preparation failed is not
  started.
- an unchanged run-once process is not run again, and an unchanged
  scheduled one is kept up by its timer: both were "a daemon that had
  stopped" and were started on every apply.
This commit is contained in:
jochen
2026-10-04 00:48:20 +02:00
parent a7bf0f6e39
commit 161ac1ff01
5 changed files with 636 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apply/apply.go
+144
View File
@@ -247,6 +247,20 @@ func ApplyKeeping(
// orphan is removed, and if a removal then fails the guard is already up. A stale opening on an
// adopted node is removed as any orphan is.
var protecting, orphans []store.Applied
// **What a declared process replaces is handed over, not removed first** (novox/hq issue 213).
// Every other orphan goes before anything is applied; one a process names under `replaces` is
// kept until that process is applied and still running a moment later, so whatever it was —
// the controller's container — answers until its replacement does, and goes on answering if
// the replacement never comes up.
replacedBy := map[string]string{}
for _, r := range d.Resources {
if p, ok := r.(*declaration.Process); ok {
for _, id := range p.Replaces {
replacedBy[id] = p.ID
}
}
}
handover := map[string][]store.Applied{}
for _, orphan := range known.Orphans(declared, origin) {
if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
protecting = append(protecting, orphan)
@@ -264,6 +278,10 @@ func ApplyKeeping(
log(fmt.Sprintf(" kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
continue
}
if by, replaced := replacedBy[orphan.ID]; replaced {
handover[by] = append(handover[by], orphan)
continue
}
orphans = append(orphans, orphan)
}
ordered := d.Resources
@@ -517,6 +535,11 @@ func ApplyKeeping(
if c, ok := resource.(*declaration.Container); ok && c.RunOnce {
gates = true
}
// And a run-once process, which is the same step hosted as a unit: a version whose
// preparation did not complete must not be started (novox/hq ADR 0135, issue 213).
if p, ok := resource.(*declaration.Process); ok && p.RunOnce {
gates = true
}
if gates {
failed.Gated = true
// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
@@ -607,6 +630,28 @@ func ApplyKeeping(
}
log(line)
}
// Its replacement applied: what it replaces goes now, once it is seen running (issue 213).
if waiting, has := handover[resource.Identity()]; has {
delete(handover, resource.Identity())
if err := handOver(ctx, run, resource, waiting, removeOrphan, &report, log); err != nil {
failures = append(failures, err)
}
}
}
// A replacement that did not apply — failed, skipped behind its module's step, held — leaves
// what it replaces running and recorded, said, for the next apply to hand over.
unhanded := make([]string, 0, len(handover))
for by := range handover {
unhanded = append(unhanded, by)
}
sort.Strings(unhanded)
for _, by := range unhanded {
for _, orphan := range handover[by] {
keptForReplacement(&report, log, orphan,
fmt.Sprintf("kept: %s, which replaces it, did not apply", by))
}
}
if !orphansRemoved {
@@ -2453,3 +2498,102 @@ func moduleOf(identity string) (string, bool) {
}
return identity[:at], true
}
// handOver removes what a process replaces, once the process is running and still is a moment later
// (novox/hq issue 213, ADR 0184). A replacement that is not up keeps what it replaces in place
// and recorded, and is this apply's failure: the old one answers until a later apply finds the new
// one up.
func handOver(ctx context.Context, run Runner, resource declaration.Resource,
waiting []store.Applied, removeOrphan func(store.Applied) error, report *Report,
log func(string)) *Error {
p, ok := resource.(*declaration.Process)
if !ok {
return nil
}
if why := stillUp(ctx, run, p.Name+".service"); why != "" {
for _, orphan := range waiting {
keptForReplacement(report, log, orphan,
fmt.Sprintf("kept: %s, which replaces it, is not running (%s)", p.ID, why))
}
return &Error{Resource: p.ID, Err: fmt.Errorf(
"%s was started and is not running a moment later (%s), so what it replaces was kept: %s",
p.Name, why, appliedIDs(waiting)), Done: *report}
}
for _, orphan := range waiting {
if err := removeOrphan(orphan); err != nil {
var failed *Error
if errors.As(err, &failed) {
return failed
}
return &Error{Resource: orphan.ID, Err: err, Done: *report}
}
}
return nil
}
// handoverSettle is how long a replacement must stay up before what it replaces goes. Longer than a
// service's second look (serviceSettle): this one decides whether the last thing answering is
// removed, and a process that fails on its store or its bus does so after it opened them, not in the
// first instant. A test sets it to nothing.
var handoverSettle = 10 * time.Second
// stillUp says why a process's unit is not up and staying up, or nothing when it is: active and
// running at two looks handoverSettle apart, the same main process both times, restarted by
// nothing in between. Stricter than stayedRunning, which reads "activating" as running — the state
// a crash-looping unit is in while it waits to be started again, which is exactly the replacement
// that must not be handed anything.
func stillUp(ctx context.Context, run Runner, unit string) string {
look := func() (map[string]string, string) {
out, err := run(ctx, "systemctl", "show", unit, "--property=ActiveState", "--property=SubState",
"--property=MainPID", "--property=NRestarts")
if err != nil {
return nil, err.Error()
}
got := map[string]string{}
for _, line := range strings.Split(out, "\n") {
if key, value, found := strings.Cut(strings.TrimSpace(line), "="); found {
got[key] = value
}
}
if got["ActiveState"] != "active" || got["SubState"] != "running" {
return got, fmt.Sprintf("%s/%s", got["ActiveState"], got["SubState"])
}
return got, ""
}
first, why := look()
if why != "" {
return why
}
timer := time.NewTimer(handoverSettle)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err().Error()
case <-timer.C:
}
second, why := look()
if why != "" {
return why
}
if first["MainPID"] != second["MainPID"] || first["NRestarts"] != second["NRestarts"] {
return fmt.Sprintf("restarted while it was watched (pid %s → %s, restarts %s → %s)",
first["MainPID"], second["MainPID"], first["NRestarts"], second["NRestarts"])
}
return ""
}
// keptForReplacement reports an orphan kept because what replaces it is not yet in its place.
func keptForReplacement(report *Report, log func(string), orphan store.Applied, detail string) {
report.Outcomes = append(report.Outcomes, Outcome{
ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "kept", Detail: detail,
})
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, strings.TrimPrefix(detail, "kept: ")))
}
func appliedIDs(applied []store.Applied) string {
ids := make([]string, 0, len(applied))
for _, a := range applied {
ids = append(ids, a.ID)
}
return strings.Join(ids, ", ")
}