package main import ( "context" "fmt" "io" "os" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // The assignment generation a declaration was composed from, and the record of every send (novox/hq // issue 234). // // On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the // assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and // cannot tell a later send that carries an older view of the assignments. So a declaration now carries // the generation of the assignments it was composed from — a counter the store raises in the same // transaction as every assignment change — and a machine refuses one older than it applied, unless it is // a gate's put-back. And every send is written down with who sent it, from which generation and naming // which modules: the controller logged no send then, and which process sent the stale declaration could // not be read back from anything the mesh kept. // kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from. const kindOlderGeneration = "older-generation" // generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. const generationRefusalsSaid = advisoryQuiet // stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and // the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the // record of the send whether or not the machine is sent them. // // **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its // declaration is allotted here like any other and carries the generation as it stands — never older than // what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never // needs to send one. func (o order) stamp(d *sendable) { d.Sequence, d.Epoch = o.sequence, o.epoch d.composedFrom, d.actingEpoch = o.generation, o.acting d.Generation = 0 if o.readsGeneration && o.generation > 0 { d.Generation = o.generation } } // sentRecord is what the record of a send keeps beside its digest. type sentRecord struct { sequence int64 epoch uint64 generation int64 // toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine // whose node-engine has not said it reads one. toldGeneration int64 sender string modules []string } // sentRecordOf is a composed send's record: its sender is the caller this process acts for. func sentRecordOf(ctx context.Context, d sendable) sentRecord { return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} } // callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the // caller the controller ran this command for, or the account at the shell. func callerOf(ctx context.Context) string { if c := link.CallerIn(ctx); c != "" { return c } return link.Caller() } // senderOf is a send's sender in words: who asked, and the process and build that composed it — the // answer issue 234 could not find, because two controllers and a one-shot push were all sending then. func senderOf(caller string) string { host, _ := os.Hostname() return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version) } // namedModules are the modules a declaration names: its machine's set, less what was left out of it. func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string { out := make([]string, 0, len(plan.Modules)) for _, m := range plan.Modules { if _, left := leftOut[m.Module]; !left { out = append(out, m.Module) } } sort.Strings(out) return out } // heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender // and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that // sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the // mesh cannot attribute is the louder fact, not a quieter one. // // **And raises the mesh's counter past what the machine applied, when the refused send was composed from // the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind // the machine, which is a store put back from a backup — and every send after would be refused for ever. A // stale sender's generation is below the counter, and the counter is left alone for it. // // Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that // loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185. // S20 names `push ` for that case instead. func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { r := report.OlderGeneration if r == nil || inv == nil || report.Node == "" { return } node, err := inv.NodeByName(ctx, report.Node) if err != nil { fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+ "read, so it is not raised: %v\n", report.Node, err) return } var raised int64 if now, err := inv.AssignmentGeneration(ctx); err != nil { fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+ "read: %v\n", report.Node, err) } else if counterBehind(*r, now) { if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil { fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+ "and could not be raised: %v\n", now, report.Node, r.Applied, err) raised = 0 } else { fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+ "a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n", now, report.Node, r.Applied, raised, report.Node) } } send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised) if err == nil && !found { send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence, Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied, CounterRaisedTo: raised}) } if err != nil { fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ "kept, so it is not raised: %v\n", report.Node, report.Sequence, err) return } fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) } // counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the // refused send carried the counter as it stands now, and the machine applied more than that. func counterBehind(r link.GenerationRefusal, now int64) bool { return r.Generation >= now && r.Applied > r.Generation } // watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was // composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal. func watchGenerationRefusals(f *signalFacts) []conditions.Observation { seen := map[string]bool{} var out []conditions.Observation for _, s := range f.refusedSends { if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] { continue } seen[s.NodeName] = true o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, Machine: s.NodeName, Severity: conditions.Warning, Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ "past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, modulesWords(s.Modules)), Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)), Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)} if s.CounterRaisedTo > 0 { // Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and // was raised; nothing has sent the machine its declaration since, so a person is asked to. o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+ "the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+ "put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo, s.NodeName) o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+ "update and kept what it had. The record is repaired; a push sends the update again.", s.NodeName, s.NodeName, s.NodeName) } out = append(out, o) } return out } // modulesWords is a send's modules as a sentence says them. func modulesWords(modules []string) string { if len(modules) == 0 { return "no module the mesh recorded" } return strings.Join(modules, ", ") } // writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and // what it answered (issue 485): nothing when the mesh has not recorded a send to it. func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { s, found, err := inv.LastSend(ctx, node) if err != nil || !found { return err } fmt.Fprintf(w, "%s:\n", node) for _, line := range lastSendLines(s) { fmt.Fprintln(w, line) } return nil } // lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when, // by whom, under which lease epoch, from which assignment generation, naming which modules, and what the // machine answered of it — the refusal in its own words. func lastSendLines(s inventory.Send) []string { const stamp = "2006-01-02 15:04:05" epoch := "under no lease epoch" if s.Epoch > 0 { epoch = fmt.Sprintf("under lease epoch %d", s.Epoch) } generation := "no assignment generation recorded" if s.Generation > 0 { generation = fmt.Sprintf("assignment generation %d", s.Generation) } lines := []string{ fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)), fmt.Sprintf(" by %s", s.Sender), fmt.Sprintf(" %s, composed from %s", epoch, generation), fmt.Sprintf(" naming %s", modulesWords(s.Modules)), } answer := func(words string) { lines = append(lines, " its answer "+words) } switch a := s.Answer; { case s.RefusedAt != nil: // Before the report: a refusal for the generation is kept on the send itself and is the machine's // answer to this send in particular. answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one", s.RefusedAt.Local().Format(stamp), s.RefusedApplied)) case a.Outcome == "" || a.At == nil: answer("the machine has not reported on it yet") case a.Outcome == inventory.OutcomeApplied: answer("applied it at " + a.At.Local().Format(stamp)) case a.Outcome == inventory.OutcomeRefused: answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:") for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") { lines = append(lines, " "+strings.TrimSpace(line)) } default: answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome, a.At.Local().Format(stamp), a.Failed)) } return lines } // sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line. func sendAnswerWord(s inventory.Send) string { switch a := s.Answer; { case s.RefusedAt != nil: return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied) case a.Outcome == "" || a.At == nil: return "not reported on yet" case a.Outcome == inventory.OutcomeFailed: return fmt.Sprintf("failed (%d resource(s))", a.Failed) default: return a.Outcome } } // printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment // generation and what it answered (novox/hq issue 485). `node show ` says the send in full. func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) { if unread != "" { fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread) return } if len(nodes) == 0 { return } fmt.Println("last sent, per machine:") for _, n := range nodes { s, found := sends[n.Name] if !found { fmt.Printf(" %-12s no send recorded\n", n.Name) continue } generation := "no generation" if s.Generation > 0 { generation = fmt.Sprintf("generation %d", s.Generation) } fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender, generation, sendAnswerWord(s)) } fmt.Printf("\n `node show ` says a machine's last send in full\n\n") }