mesh-controller/cmd/mesh-controller/apply_failing.go

package main

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
)

// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
// for longer than one apply interval (novox/hq issue 233).
const kindApplyFailing = "apply-failing"

// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
const applyEvery = 5 * time.Minute

// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
// (mesh-host's probe.go): "the package manager (pacman)".
const packageManagerKey = "the package manager ("

// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
//
// **The bound gates raising, never keeping** (sweep 2026-10-30, issue 774). A report says a failure again whenever its
// words change, with a new time: on novox the vault's refusal of every secret replaced a bus refusal at 16:56, and the
// condition cleared for five minutes while every module still failed. Once raised, it stays raised while any failure
// stands on that machine, judged by the open condition the keeper holds on the bus, not by a report's time.
func watchApplyFailing(f *signalFacts) []conditions.Observation {
	heard := map[string]bool{}
	for _, m := range f.machines {
		if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
			heard[m.name] = true
		}
	}
	var out []conditions.Observation
	said := map[string]bool{}
	for _, d := range f.doing {
		if !d.Wrong() || d.Since == nil || !heard[d.Node] || (f.now.Sub(*d.Since) <= applyEvery && !f.applyFailingOpen[d.Node]) {
			continue
		}
		out = append(out, applyFailing(d, f.now))
		said[d.Node] = true
	}
	// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
	// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
	// on a machine whose node-engine applied what it was sent.
	failed := map[string][]inventory.ProvReport{}
	for _, r := range f.provFailed {
		if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] ||
			(f.now.Sub(r.Since()) <= applyEvery && !f.applyFailingOpen[r.Node]) {
			continue
		}
		failed[r.Node] = append(failed[r.Node], r)
	}
	for _, node := range sortedNames(failed) {
		out = append(out, provisioningFailing(node, failed[node], f.now))
	}
	return out
}

// applyFailingOpen are the machines whose S21 condition is open among the open conditions.
func applyFailingOpen(open []conditions.Condition) map[string]bool {
	out := map[string]bool{}
	for _, c := range open {
		if c.Kind == kindApplyFailing && c.Subject.Machine != "" {
			out[c.Subject.Machine] = true
		}
	}
	return out
}

// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
// place.
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
	sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
	var names, words []string
	since := failed[0].Since()
	for _, r := range failed {
		names = append(names, r.Module)
		words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
		if r.Since().Before(since) {
			since = r.Since()
		}
	}
	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
		Severity: conditions.Warning, Said: strings.Join(words, "; "),
		Resolved: m + " applies what it is sent again"}
	o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
		strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
	o.Headline = m + " is not applying its update"
	o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
		"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
	return o
}

func orNoReason(s string) string {
	if s == "" {
		return "no reason given"
	}
	return s
}

// applyFailing is one machine's S21 condition.
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
	m := d.Node
	since := d.Since.UTC().Format("2006-01-02 15:04 MST")
	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
		Severity: conditions.Warning, Said: doingWords(d),
		Resolved: m + " applies what it is sent again"}
	how := "refused its declaration whole"
	if d.Outcome != inventory.OutcomeRefused {
		how = "applied its declaration only in part"
	}
	o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
	o.Headline = m + " is not applying its update"
	o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
		m, plainHow(d), humanDuration(now.Sub(*d.Since)))
	if tool := unansweringTool(d); tool != "" {
		o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
			"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
		o.Headline = fmt.Sprintf("%s cannot install packages", m)
		o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
			"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
			"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
	}
	return o
}

// plainHow is how a machine failed, as the operator reads it.
func plainHow(d inventory.Doing) string {
	if d.Outcome == inventory.OutcomeRefused {
		return "refused every update"
	}
	return "applied its updates only in part"
}

// unansweringTool is the package manager a machine said does not answer, or "".
func unansweringTool(d inventory.Doing) string {
	for _, r := range d.Failed {
		if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
			return strings.TrimSuffix(rest, ")")
		}
	}
	return ""
}

// failedNames is what failed, by name: the refusal, or the failed resources' ids.
func failedNames(d inventory.Doing) string {
	if d.Outcome == inventory.OutcomeRefused {
		return "refused"
	}
	ids := make([]string, 0, len(d.Failed))
	for _, r := range d.Failed {
		ids = append(ids, r.ID)
	}
	sort.Strings(ids)
	return "failed " + strings.Join(ids, ", ")
}

// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
func doingWords(d inventory.Doing) string {
	if d.Outcome == inventory.OutcomeRefused {
		return "refused: " + oneLine(d.Refused)
	}
	var parts []string
	for _, r := range d.Failed {
		parts = append(parts, r.ID+": "+oneLine(r.Error))
	}
	sort.Strings(parts)
	return strings.Join(parts, "; ")
}