jochen
875a4e5758
plans said a build queued for minutes had been building for a few seconds: the line counted from the last save, and every advance saved the plan whether or not it moved, bumping its revision and saying plan-moved on the bus. The line, status and LATE now count from when the plan entered its tier with the stalled condition's bound, and an advance that changes nothing writes nothing.
633 lines
26 KiB
Go
633 lines
26 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/overlay"
|
|
)
|
|
|
|
// is anything broken, is anything not answering, is anything out of date.
|
|
//
|
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
|
|
|
// short is a commit as a person refers to it.
|
|
func short(commit string) string {
|
|
if len(commit) > 8 {
|
|
return commit[:8]
|
|
}
|
|
return commit
|
|
}
|
|
|
|
// statusCommand answers "did my change go out?".
|
|
//
|
|
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
|
|
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
|
|
// this is worse to live with whatever its other properties.
|
|
//
|
|
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
|
|
// source now has, and which machines are running the old one.
|
|
func statusCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("status", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "the same answers, for something other than a person")
|
|
if _, err := parseAround(set, args); err != nil {
|
|
return err
|
|
}
|
|
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return statusFor(ctx, open, *asJSON)
|
|
}
|
|
|
|
// statusFor asks and answers, against stores somebody else opened.
|
|
//
|
|
// Split from the command so what it prints can be read by a test. The sentence it prints when nothing
|
|
// is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact
|
|
// words the thing worth holding still.
|
|
func statusFor(ctx context.Context, open *stores, asJSON bool) error {
|
|
asked, err := theThreeQuestions(ctx, open)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if asJSON {
|
|
body, err := statusAsJSON(asked)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(string(body))
|
|
return nil
|
|
}
|
|
return printStatus(asked)
|
|
}
|
|
|
|
// printStatus is the words, separated from the questions.
|
|
//
|
|
// **Its exact sentences have been acted on and been misleading twice** — a held module reading as a
|
|
// machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being
|
|
// true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a
|
|
// test can read them without a store, a bus or a machine.
|
|
func printStatus(asked answers) error {
|
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
|
behind, sources := asked.behind, asked.sources
|
|
|
|
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
|
// first, silenced ones with when their silence ends.
|
|
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
|
|
|
if len(asked.refused) > 0 {
|
|
// First of what follows. A machine that cannot be worked out is not running an old
|
|
// declaration — it has no declaration, and nothing below this line is about it.
|
|
var names []string
|
|
for name := range asked.refused {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
|
|
len(names))
|
|
for _, name := range names {
|
|
fmt.Printf(" %s\n", name)
|
|
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
|
fmt.Printf(" %s\n", strings.TrimSpace(line))
|
|
}
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
if asked.network != "" {
|
|
fmt.Printf("the private network could not be computed:\n %s\n\n",
|
|
strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
|
|
}
|
|
|
|
if len(wrong) > 0 {
|
|
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
|
|
for _, d := range wrong {
|
|
fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04"))
|
|
if d.Refused != "" {
|
|
// The host's own words. It says exactly what it could not accept, and nothing
|
|
// written here would say it better.
|
|
fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused))
|
|
}
|
|
for _, f := range d.Failed {
|
|
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
|
|
}
|
|
if d.Stuck() {
|
|
// Said apart from the failure itself. The host's words say what is wrong; this
|
|
// says it is not new — the machine has applied, failed the same way and reported
|
|
// so this many times, and will keep doing exactly that until something changes
|
|
// (novox/hq 04-ISSUES/065).
|
|
fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
|
|
"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
|
|
}
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
if len(quiet) > 0 {
|
|
var said []string
|
|
for _, n := range quiet {
|
|
said = append(said, n.Name+" ("+heardFrom(n)+")")
|
|
}
|
|
fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n",
|
|
len(quiet), strings.Join(said, "\n "))
|
|
}
|
|
|
|
if open, late := openPlans(asked.plans, time.Now(), asked.paused, asked.tierBounds); len(open) > 0 {
|
|
fmt.Printf("%d plan(s) open", len(open))
|
|
if late > 0 {
|
|
fmt.Printf(", %d in a tier past its bound", late)
|
|
}
|
|
fmt.Println(":")
|
|
for _, p := range open {
|
|
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused, asked.tierBounds.of(p.Repository)))
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
if len(behind) > 0 {
|
|
var names []string
|
|
for m := range behind {
|
|
names = append(names, m)
|
|
}
|
|
sort.Strings(names)
|
|
|
|
fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
|
|
for _, m := range names {
|
|
from := sources[m]
|
|
fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
|
|
if on := behind[m]; len(on) > 0 {
|
|
// The part somebody actually wants. A module being out of date is a fact about
|
|
// the catalogue; machines running the old one is the thing with consequences.
|
|
fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", "))
|
|
} else {
|
|
fmt.Printf(" %-18s assigned to nothing\n", "")
|
|
}
|
|
}
|
|
// The remedy, beside the problem. A status that says what is wrong and not what to do
|
|
// about it makes somebody go and find the command, and the command is the whole point of
|
|
// having noticed.
|
|
fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n")
|
|
fmt.Println()
|
|
}
|
|
|
|
if len(asked.waiting) > 0 {
|
|
// The other half of "is anything out of date": a module behind its source says the
|
|
// catalogue is old, and this says a machine is — and only this one has somebody's change
|
|
// waiting inside it.
|
|
var told, never []string
|
|
for _, m := range asked.waiting {
|
|
if m.Never {
|
|
never = append(never, m.Node)
|
|
continue
|
|
}
|
|
told = append(told, m.Node)
|
|
}
|
|
if len(told) > 0 {
|
|
fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n",
|
|
len(told), strings.Join(told, ", "))
|
|
}
|
|
if len(never) > 0 {
|
|
// Never told is not out of date. The remedy is the same push and the situation is
|
|
// not the same at all: nobody has ever asked this machine to be anything.
|
|
fmt.Printf("%d machine(s) have never been sent anything:\n %s\n",
|
|
len(never), strings.Join(never, ", "))
|
|
}
|
|
fmt.Printf("\n `push --behind` sends them\n\n")
|
|
}
|
|
|
|
if split := hostSplit(nodes); len(split) > 1 {
|
|
// **Before a declaration gains a field, every machine has to understand it** (novox/hq
|
|
// 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses
|
|
// it whole, so every new field is a flag day: hosts first, then the controller. The mesh had
|
|
// no record of which host any machine ran, so that order was kept by somebody remembering it.
|
|
//
|
|
// **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and
|
|
// commits have no order — the first version of this said "N machines run an older host" and
|
|
// named the three that were newer, because it compared two hashes as strings. What the mesh
|
|
// can say truthfully is that the machines do not all run the same host, and which machines
|
|
// hold which. Ordering needs a version that is ordered, and that is the host's to report.
|
|
versions := make([]string, 0, len(split))
|
|
for v := range split {
|
|
versions = append(versions, v)
|
|
}
|
|
sort.Strings(versions)
|
|
fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes))
|
|
for _, v := range versions {
|
|
sort.Strings(split[v])
|
|
fmt.Printf(" %-12s %s\n", v, strings.Join(split[v], ", "))
|
|
}
|
|
fmt.Printf("\n a host refuses a declaration carrying a field it does not know, whole — so the\n" +
|
|
" mesh may send only what every one of these understands. Which of them is newer is\n" +
|
|
" not readable from a commit; that needs a version the host reports as ordered\n\n")
|
|
}
|
|
|
|
if len(asked.filtered) > 0 {
|
|
// A converged machine is filtered by the mesh alone, and the mesh says truthfully which
|
|
// (novox/hq ADR 0168). One that is not — a predecessor's chain still refusing, a found
|
|
// firewall in force again — is said here, and is not well.
|
|
machines := make([]string, 0, len(asked.filtered))
|
|
for name := range asked.filtered {
|
|
machines = append(machines, name)
|
|
}
|
|
sort.Strings(machines)
|
|
fmt.Printf("%d converged machine(s) are not filtered by the mesh alone:\n", len(machines))
|
|
for _, name := range machines {
|
|
f := asked.filtered[name]
|
|
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
|
fmt.Printf(" %-12s the found firewall (%s) is in force again; the next apply retires it\n", name, fw.Kind)
|
|
}
|
|
for _, x := range f.Others() {
|
|
fmt.Printf(" %-12s %s (%s): %s\n", name, x.Where, x.Owner, x.Refuses)
|
|
}
|
|
}
|
|
fmt.Printf("\n the mesh wrote none of these and removes none; `node show <node>` lists every filter with its owner\n\n")
|
|
}
|
|
|
|
if len(asked.untaken) > 0 {
|
|
// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
|
|
// somebody chose and can leave alone; a module assigned to one and never taken is work
|
|
// outstanding that reads exactly like work finished. That reading is what stopped a
|
|
// predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125).
|
|
machines := make([]string, 0, len(asked.untaken))
|
|
for name := range asked.untaken {
|
|
machines = append(machines, name)
|
|
}
|
|
sort.Strings(machines)
|
|
total := 0
|
|
for _, held := range asked.untaken {
|
|
for _, n := range held {
|
|
total += n
|
|
}
|
|
}
|
|
fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+
|
|
"taken — so it is running none of what it declares:\n", total)
|
|
for _, name := range machines {
|
|
modules := make([]string, 0, len(asked.untaken[name]))
|
|
for m := range asked.untaken[name] {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
parts := make([]string, 0, len(modules))
|
|
for _, m := range modules {
|
|
parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m]))
|
|
}
|
|
fmt.Printf(" %-12s %s\n", name, strings.Join(parts, ", "))
|
|
}
|
|
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
|
}
|
|
|
|
if len(asked.unheld) > 0 {
|
|
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
|
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
|
// holds, until every machine has its holders and the switch makes it a refusal.
|
|
fmt.Printf("%d module dependenc(ies) on a seat nothing on the machine holds (unheld, ADR 0207):\n",
|
|
len(asked.unheld))
|
|
for _, u := range asked.unheld {
|
|
holders := "no module in the catalogue claims it yet"
|
|
if len(u.Holders) > 0 {
|
|
holders = "could be held by " + strings.Join(u.Holders, ", ")
|
|
}
|
|
fmt.Printf(" %-12s %-24s %-24s %s\n", u.Node, u.Module, u.Seat, holders)
|
|
}
|
|
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
|
}
|
|
|
|
if len(asked.overflowing) > 0 {
|
|
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
|
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
|
// machine is sent everything else rather than refused for one consumer elsewhere.
|
|
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
|
len(asked.overflowing))
|
|
for _, o := range asked.overflowing {
|
|
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
|
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
|
}
|
|
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
|
}
|
|
|
|
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
|
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
|
switch {
|
|
case asked.handActsUnread != "":
|
|
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
|
asked.handActsUnread)
|
|
case asked.handActs != nil && *asked.handActs > 0:
|
|
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
|
}
|
|
|
|
// And what the mesh repaired by itself (novox/hq to-be 45 §7): seen, not only done.
|
|
switch {
|
|
case asked.healsUnread != "":
|
|
fmt.Printf("what the healers did could not be read: %s\n\n", asked.healsUnread)
|
|
case asked.heals != nil && (asked.heals.Acts > 0 || asked.heals.Escalated > 0):
|
|
fmt.Printf("%d repair(s) made by the healers in the last seven days, %d handed to the operator — `healers` "+
|
|
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
|
}
|
|
|
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
|
fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
|
|
fmt.Printf("\n `converge <node>` previews the flip\n\n")
|
|
}
|
|
|
|
if asked.well() {
|
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
|
// and getting here means every question was asked and answered. **No open conditions first**
|
|
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
|
|
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
|
|
"the mesh would send them, and every module current with its source\n", len(nodes))
|
|
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
|
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
|
// about the relationship between the mesh and a machine — applied what it was sent, matches
|
|
// what would be sent, built from what the source has. None of them asks whether a module can
|
|
// reach what it requires, and the mesh composes every one of those grants itself.
|
|
//
|
|
// Said here rather than left to be inferred. A reader who acts on the line above is acting on
|
|
// "the machines are as the mesh described them", and the distance between that and "it works"
|
|
// is where the eleven hours went.
|
|
fmt.Printf("\n That is the mesh and the machines agreeing. Nothing here dials a provision:\n" +
|
|
" no grant the mesh composed has been tested, so a module unable to reach what it\n" +
|
|
" requires would not appear above (04-ISSUES/145)\n")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// firstLine is as much of a failure as belongs in a list.
|
|
func firstLine(s string) string {
|
|
if cut := strings.IndexByte(s, '\n'); cut >= 0 {
|
|
return strings.TrimSpace(s[:cut])
|
|
}
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
// builds keeps what a builder said, for the serving control plane.
|
|
//
|
|
// A type of its own rather than a method on the enrolment, because they are unrelated things
|
|
// arriving on one queue and an implementation of one should not have to say anything about the
|
|
// other.
|
|
type builds struct {
|
|
inv *inventory.Inventory
|
|
open *stores
|
|
}
|
|
|
|
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
|
|
//
|
|
// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There
|
|
// are three now — a person's status, its JSON, and a page — and three implementations of "which
|
|
// machine is not doing what it was told" would be three chances to disagree about it.
|
|
//
|
|
// The order is the design and not a convenience: is anything broken, is anything not answering, is
|
|
// anything out of date. The first has consequences now, the second may, the third is a plan for
|
|
// later — and anything that led with the third would bury the first.
|
|
func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|
inv := open.inventory
|
|
var out answers
|
|
var err error
|
|
|
|
out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.nodes, err = inv.Nodes(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
for _, n := range out.nodes {
|
|
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
|
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
|
// and could not.
|
|
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
|
out.quiet = append(out.quiet, n)
|
|
}
|
|
}
|
|
out.behind, err = inv.Behind(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
// And why any machine cannot be worked out at all, which is neither of the first two questions
|
|
// and is asked before them both in practice: a machine nothing can be computed for is not
|
|
// broken, not quiet and not behind, and every other answer here would call it well.
|
|
//
|
|
// Read through whoResolves, which is what the private network is built from, so this and the
|
|
// network agree about who could not be resolved rather than deciding it twice.
|
|
_, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
// And what each machine is holding rather than running, by the module that would run it. Read
|
|
// from what the machine itself last reported, not from what take-time computed: the machine is
|
|
// the only thing that knows what it found (novox/hq 04-ISSUES/125).
|
|
out.untaken, err = untakenModules(ctx, inv, out.nodes)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
// And which converged machines something other than the mesh filters (novox/hq ADR 0168), as
|
|
// each last reported — the account that was missing when a predecessor's chain refused what the
|
|
// mesh declared open for eleven hours (04-ISSUES/144, 145).
|
|
out.filtered, err = filteredMachines(ctx, inv, out.nodes)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
// And which machines run a module whose resources a seat nothing there holds applies (novox/hq
|
|
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
|
// the private network is built from and should say nothing else; a machine that does not
|
|
// resolve is already in refused, and is passed over here.
|
|
plans := map[string]planned{}
|
|
for _, n := range out.nodes {
|
|
plan, settings, err := planFor(ctx, open, n.Name)
|
|
if err != nil {
|
|
if unresolvable(err) {
|
|
continue
|
|
}
|
|
return answers{}, err
|
|
}
|
|
plans[n.Name] = planned{plan, settings}
|
|
out.unheld = append(out.unheld, plan.Unheld...)
|
|
// And which of its modules a provider leaves out of its grants, for an identity too long
|
|
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
|
// resolution, as the provider's composition judges it.
|
|
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
|
}
|
|
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
|
|
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
|
|
// the bus; a process that cannot read them says so, and the mesh is then not called well.
|
|
if out.conditions, err = openConditions(ctx); err != nil {
|
|
out.conditionsUnread = err.Error()
|
|
}
|
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
|
out.paused = buildSeatPause(ctx, inv, out.plans)
|
|
// And how long a tier may take, the bound a plan is late at (novox/hq issue 296).
|
|
out.tierBounds = readTierBounds(ctx, inv, time.Now())
|
|
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
|
// to read the log from; a process with none has no log to count.
|
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
|
n, unread := handActsThisWeek(ctx)
|
|
if unread != "" {
|
|
out.handActsUnread = unread
|
|
} else {
|
|
out.handActs = &n
|
|
}
|
|
}
|
|
|
|
// And what the healers did this week (novox/hq to-be 45 §7).
|
|
if heals, err := inv.HealsSince(ctx, time.Now().Add(-7*24*time.Hour)); err != nil {
|
|
out.healsUnread = err.Error()
|
|
} else {
|
|
out.heals = countHeals(heals)
|
|
}
|
|
|
|
// And which machines are not running what the mesh would send them. The same question as a
|
|
// module being behind its source, one level down: that one says the catalogue is out of date,
|
|
// this one says a machine is — and only the second has anybody's change waiting in it.
|
|
//
|
|
// **One machine that cannot be resolved must not take the answer away from every other**
|
|
// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
|
|
// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
|
|
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
|
// reason is kept and reported as data; every question that does not depend on it is still
|
|
// answered.
|
|
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
|
if err != nil {
|
|
out.network = err.Error()
|
|
would = map[string]string{}
|
|
}
|
|
out.waiting, err = inv.Waiting(ctx, would)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.reported, err = inv.LastReports(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.sources = map[string]inventory.Source{}
|
|
for module := range out.behind {
|
|
from, err := inv.SourceOf(ctx, module)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.sources[module] = from
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// untakenModules is, per machine, each module whose resources that machine is holding as found, and
|
|
// how many.
|
|
//
|
|
// **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found
|
|
// and reports it; the mesh's take-time listing is a different thing and was the one this command used
|
|
// to have, which is why a module assigned after the listing showed nothing at all
|
|
// (novox/hq 04-ISSUES/125).
|
|
//
|
|
// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
|
|
// the caller prints nothing.
|
|
// filteredMachines is every converged machine not filtered by the mesh alone, with what it last said
|
|
// filters it (novox/hq ADR 0168). An adopted machine keeps its found firewall by design and is not
|
|
// counted; a machine that has not said is not said to be filtered by anything.
|
|
func filteredMachines(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
|
map[string]inventory.Filtering, error) {
|
|
out := map[string]inventory.Filtering{}
|
|
for _, n := range nodes {
|
|
if n.Adopted {
|
|
continue
|
|
}
|
|
f, err := inv.FilteringOf(ctx, n.Name)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("what filters %s cannot be read: %w", n.Name, err)
|
|
}
|
|
if len(f.Filters) == 0 && f.FoundFirewall == nil {
|
|
continue
|
|
}
|
|
if !f.Alone() {
|
|
out[n.Name] = f
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
|
map[string]map[string]int, error) {
|
|
|
|
out := map[string]map[string]int{}
|
|
for _, n := range nodes {
|
|
said, err := inv.AdoptionOf(ctx, n.Name)
|
|
if err != nil {
|
|
// A machine whose record cannot be read is not a machine holding nothing. Said, because
|
|
// answering "nothing held" from a failed read is the shape this whole issue is about.
|
|
return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err)
|
|
}
|
|
for _, h := range said.Held {
|
|
if h.Module == "" {
|
|
continue // a hold the mesh cannot attribute to a module has nothing to take
|
|
}
|
|
if out[n.Name] == nil {
|
|
out[n.Name] = map[string]int{}
|
|
}
|
|
out[n.Name][h.Module]++
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// well is whether every question this command asks came back with nothing to say.
|
|
//
|
|
// Named, and in one place, because it is the sentence an operator acts on and it has been wrong
|
|
// twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken
|
|
// and is not doing what it was told either.
|
|
//
|
|
// **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave
|
|
// alone. A module assigned to a machine and never taken is a half-finished action with nothing left
|
|
// to finish it — it runs none of what it declares, and "all doing what they were told" was true and
|
|
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
|
func (a answers) well() bool {
|
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
|
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
|
len(a.conditions) == 0 && a.conditionsUnread == ""
|
|
}
|
|
|
|
// hostSplit is which machines report which host version, for every version more than one machine
|
|
// could disagree about.
|
|
//
|
|
// **It does not say which is newer, because it cannot.** A host reports its version as a commit, and
|
|
// commits have no order. The first version of this returned "the machines behind the newest" by
|
|
// comparing versions as strings, and on the live mesh it named the three machines running the NEWER
|
|
// host as the ones behind — an arbitrary lexicographic result presented as a fact
|
|
// (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one
|
|
// that says less, which is the whole subject of 04-ISSUES/145.
|
|
//
|
|
// So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no
|
|
// ordering. Ordering wants an ordered version, and that is the host's to report rather than this
|
|
// function's to infer.
|
|
//
|
|
// Machines that have not reported a version are left out entirely: they are not a version, and
|
|
// counting them as one would invent a disagreement. `node show` says per machine that it has not said.
|
|
func hostSplit(nodes []inventory.Node) map[string][]string {
|
|
out := map[string][]string{}
|
|
for _, n := range nodes {
|
|
if n.HostVersion == "" {
|
|
continue
|
|
}
|
|
out[n.HostVersion] = append(out[n.HostVersion], n.Name)
|
|
}
|
|
if len(out) < 2 {
|
|
return nil // one version, or none reported: nothing to disagree about
|
|
}
|
|
return out
|
|
}
|