mesh-controller/cmd/mesh-controller/generation.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// The assignment generation a declaration was composed from, and the record of every send (novox/hq
// issue 234).
//
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
// the generation of the assignments it was composed from — a counter the store raises in the same
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
// which modules: the controller logged no send then, and which process sent the stale declaration could
// not be read back from anything the mesh kept.

// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
const kindOlderGeneration = "older-generation"

// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
const generationRefusalsSaid = advisoryQuiet

// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
// record of the send whether or not the machine is sent them.
//
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
// declaration is allotted here like any other and carries the generation as it stands — never older than
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
// needs to send one.
func (o order) stamp(d *sendable) {
	d.Sequence, d.Epoch = o.sequence, o.epoch
	d.composedFrom, d.actingEpoch = o.generation, o.acting
	d.Generation = 0
	if o.readsGeneration && o.generation > 0 {
		d.Generation = o.generation
	}
}

// sentRecord is what the record of a send keeps beside its digest.
type sentRecord struct {
	sequence   int64
	epoch      uint64
	generation int64
	// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
	// whose node-engine has not said it reads one.
	toldGeneration int64
	sender         string
	modules        []string
	// plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none.
	plans []byte
}

// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
	r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
		toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
	if len(d.plans) > 0 {
		kept := map[string]keptPlan{}
		for module, p := range d.plans {
			kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]}
		}
		// A map of plain strings and slices: it marshals.
		r.plans, _ = json.Marshal(kept)
	}
	return r
}

// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in
// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans).
type keptPlan struct {
	catalogue.Plan
	RequiredBy string `json:"required-by,omitempty"`
}

// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
// caller the controller ran this command for, or the account at the shell.
func callerOf(ctx context.Context) string {
	if c := link.CallerIn(ctx); c != "" {
		return c
	}
	return link.Caller()
}

// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
func senderOf(caller string) string {
	host, _ := os.Hostname()
	return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
}

// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
	out := make([]string, 0, len(plan.Modules))
	for _, m := range plan.Modules {
		if _, left := leftOut[m.Module]; !left {
			out = append(out, m.Module)
		}
	}
	sort.Strings(out)
	return out
}

// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
// mesh cannot attribute is the louder fact, not a quieter one.
//
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
// stale sender's generation is below the counter, and the counter is left alone for it.
//
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
// S20 names `push <node>` for that case instead.
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
	r := report.OlderGeneration
	if r == nil || inv == nil || report.Node == "" {
		return
	}
	node, err := inv.NodeByName(ctx, report.Node)
	if err != nil {
		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
			"read, so it is not raised: %v\n", report.Node, err)
		return
	}
	var raised int64
	if now, err := inv.AssignmentGeneration(ctx); err != nil {
		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
			"read: %v\n", report.Node, err)
	} else if counterBehind(*r, now) {
		if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
			fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
				"and could not be raised: %v\n", now, report.Node, r.Applied, err)
			raised = 0
		} else {
			fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
				"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
				now, report.Node, r.Applied, raised, report.Node)
		}
	}
	send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
	if err == nil && !found {
		send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
			Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
			CounterRaisedTo: raised})
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
			"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
		return
	}
	fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
		"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
}

// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
// refused send carried the counter as it stands now, and the machine applied more than that.
func counterBehind(r link.GenerationRefusal, now int64) bool {
	return r.Generation >= now && r.Applied > r.Generation
}

// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
	seen := map[string]bool{}
	var out []conditions.Observation
	for _, s := range f.refusedSends {
		if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
			continue
		}
		seen[s.NodeName] = true
		o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
			Machine: s.NodeName, Severity: conditions.Warning,
			Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
				"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
				"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
				modulesWords(s.Modules)),
			Said:     fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
			Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
			Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
				"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
			Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
		if s.CounterRaisedTo > 0 {
			// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
			// was raised; nothing has sent the machine its declaration since, so a person is asked to.
			o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
				"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
				"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
				s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
				s.NodeName)
			o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
				"update and kept what it had. The record is repaired; a push sends the update again.",
				s.NodeName, s.NodeName, s.NodeName)
		}
		out = append(out, o)
	}
	return out
}

// modulesWords is a send's modules as a sentence says them.
func modulesWords(modules []string) string {
	if len(modules) == 0 {
		return "no module the mesh recorded"
	}
	return strings.Join(modules, ", ")
}

// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
	s, found, err := inv.LastSend(ctx, node)
	if err != nil || !found {
		return err
	}
	fmt.Fprintf(w, "%s:\n", node)
	for _, line := range lastSendLines(s) {
		fmt.Fprintln(w, line)
	}
	return nil
}

// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
// machine answered of it — the refusal in its own words.
func lastSendLines(s inventory.Send) []string {
	const stamp = "2006-01-02 15:04:05"
	epoch := "under no lease epoch"
	if s.Epoch > 0 {
		epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
	}
	generation := "no assignment generation recorded"
	if s.Generation > 0 {
		generation = fmt.Sprintf("assignment generation %d", s.Generation)
	}
	lines := []string{
		fmt.Sprintf("  last sent         sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
		fmt.Sprintf("                    by %s", s.Sender),
		fmt.Sprintf("                    %s, composed from %s", epoch, generation),
		fmt.Sprintf("                    naming %s", modulesWords(s.Modules)),
	}
	answer := func(words string) { lines = append(lines, "  its answer        "+words) }
	switch a := s.Answer; {
	case s.RefusedAt != nil:
		// Before the report: a refusal for the generation is kept on the send itself and is the machine's
		// answer to this send in particular.
		answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
			s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
	case a.Outcome == "" || a.At == nil:
		answer("the machine has not reported on it yet")
	case a.Outcome == inventory.OutcomeApplied:
		answer("applied it at " + a.At.Local().Format(stamp))
	case a.Outcome == inventory.OutcomeRefused:
		answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
		for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
			lines = append(lines, "                      "+strings.TrimSpace(line))
		}
	default:
		answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
			a.At.Local().Format(stamp), a.Failed))
	}
	return lines
}

// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
func sendAnswerWord(s inventory.Send) string {
	switch a := s.Answer; {
	case s.RefusedAt != nil:
		return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
	case a.Outcome == "" || a.At == nil:
		return "not reported on yet"
	case a.Outcome == inventory.OutcomeFailed:
		return fmt.Sprintf("failed (%d resource(s))", a.Failed)
	default:
		return a.Outcome
	}
}

// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
	if unread != "" {
		fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
		return
	}
	if len(nodes) == 0 {
		return
	}
	fmt.Println("last sent, per machine:")
	for _, n := range nodes {
		s, found := sends[n.Name]
		if !found {
			fmt.Printf("  %-12s no send recorded\n", n.Name)
			continue
		}
		generation := "no generation"
		if s.Generation > 0 {
			generation = fmt.Sprintf("generation %d", s.Generation)
		}
		fmt.Printf("  %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
			generation, sendAnswerWord(s))
	}
	fmt.Printf("\n  `node show <node>` says a machine's last send in full\n\n")
}