284 lines
10 KiB
Go
284 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/overlay"
|
|
)
|
|
|
|
// is anything broken, is anything not answering, is anything out of date.
|
|
//
|
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
|
|
|
// short is a commit as a person refers to it.
|
|
func short(commit string) string {
|
|
if len(commit) > 8 {
|
|
return commit[:8]
|
|
}
|
|
return commit
|
|
}
|
|
|
|
// statusCommand answers "did my change go out?".
|
|
//
|
|
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
|
|
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
|
|
// this is worse to live with whatever its other properties.
|
|
//
|
|
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
|
|
// source now has, and which machines are running the old one.
|
|
func statusCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("status", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "the same answers, for something other than a person")
|
|
if _, err := parseAround(set, args); err != nil {
|
|
return err
|
|
}
|
|
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
|
|
asked, err := theThreeQuestions(ctx, open)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
|
behind, sources := asked.behind, asked.sources
|
|
|
|
if *asJSON {
|
|
body, err := statusAsJSON(asked)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(string(body))
|
|
return nil
|
|
}
|
|
|
|
if len(asked.refused) > 0 {
|
|
// First, above everything else. A machine that cannot be worked out is not running an old
|
|
// declaration — it has no declaration, and nothing below this line is about it.
|
|
var names []string
|
|
for name := range asked.refused {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
|
|
len(names))
|
|
for _, name := range names {
|
|
fmt.Printf(" %s\n", name)
|
|
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
|
fmt.Printf(" %s\n", strings.TrimSpace(line))
|
|
}
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
if asked.network != "" {
|
|
fmt.Printf("the private network could not be computed:\n %s\n\n",
|
|
strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
|
|
}
|
|
|
|
if len(wrong) > 0 {
|
|
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
|
|
for _, d := range wrong {
|
|
fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04"))
|
|
if d.Refused != "" {
|
|
// The host's own words. It says exactly what it could not accept, and nothing
|
|
// written here would say it better.
|
|
fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused))
|
|
}
|
|
for _, f := range d.Failed {
|
|
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
|
|
}
|
|
if d.Stuck() {
|
|
// Said apart from the failure itself. The host's words say what is wrong; this
|
|
// says it is not new — the machine has applied, failed the same way and reported
|
|
// so this many times, and will keep doing exactly that until something changes
|
|
// (novox/hq 04-ISSUES/065).
|
|
fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
|
|
"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
|
|
}
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
if len(quiet) > 0 {
|
|
var said []string
|
|
for _, n := range quiet {
|
|
said = append(said, n.Name+" ("+heardFrom(n)+")")
|
|
}
|
|
fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n",
|
|
len(quiet), strings.Join(said, "\n "))
|
|
}
|
|
|
|
if len(behind) > 0 {
|
|
var names []string
|
|
for m := range behind {
|
|
names = append(names, m)
|
|
}
|
|
sort.Strings(names)
|
|
|
|
fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
|
|
for _, m := range names {
|
|
from := sources[m]
|
|
fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
|
|
if on := behind[m]; len(on) > 0 {
|
|
// The part somebody actually wants. A module being out of date is a fact about
|
|
// the catalogue; machines running the old one is the thing with consequences.
|
|
fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", "))
|
|
} else {
|
|
fmt.Printf(" %-18s assigned to nothing\n", "")
|
|
}
|
|
}
|
|
// The remedy, beside the problem. A status that says what is wrong and not what to do
|
|
// about it makes somebody go and find the command, and the command is the whole point of
|
|
// having noticed.
|
|
fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n")
|
|
fmt.Println()
|
|
}
|
|
|
|
if len(asked.waiting) > 0 {
|
|
// The other half of "is anything out of date": a module behind its source says the
|
|
// catalogue is old, and this says a machine is — and only this one has somebody's change
|
|
// waiting inside it.
|
|
var told, never []string
|
|
for _, m := range asked.waiting {
|
|
if m.Never {
|
|
never = append(never, m.Node)
|
|
continue
|
|
}
|
|
told = append(told, m.Node)
|
|
}
|
|
if len(told) > 0 {
|
|
fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n",
|
|
len(told), strings.Join(told, ", "))
|
|
}
|
|
if len(never) > 0 {
|
|
// Never told is not out of date. The remedy is the same push and the situation is
|
|
// not the same at all: nobody has ever asked this machine to be anything.
|
|
fmt.Printf("%d machine(s) have never been sent anything:\n %s\n",
|
|
len(never), strings.Join(never, ", "))
|
|
}
|
|
fmt.Printf("\n `push --behind` sends them\n\n")
|
|
}
|
|
|
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
|
fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
|
|
fmt.Printf("\n `converge <node>` previews the flip\n\n")
|
|
}
|
|
|
|
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
|
|
len(asked.refused) == 0 && asked.network == "" {
|
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
|
// and getting here means every question was asked and answered.
|
|
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
|
"the mesh would send them, and every module current with its source\n", len(nodes))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// firstLine is as much of a failure as belongs in a list.
|
|
func firstLine(s string) string {
|
|
if cut := strings.IndexByte(s, '\n'); cut >= 0 {
|
|
return strings.TrimSpace(s[:cut])
|
|
}
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
// builds keeps what a builder said, for the serving control plane.
|
|
//
|
|
// A type of its own rather than a method on the enrolment, because they are unrelated things
|
|
// arriving on one queue and an implementation of one should not have to say anything about the
|
|
// other.
|
|
type builds struct{ inv *inventory.Inventory }
|
|
|
|
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
|
|
//
|
|
// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There
|
|
// are three now — a person's status, its JSON, and a page — and three implementations of "which
|
|
// machine is not doing what it was told" would be three chances to disagree about it.
|
|
//
|
|
// The order is the design and not a convenience: is anything broken, is anything not answering, is
|
|
// anything out of date. The first has consequences now, the second may, the third is a plan for
|
|
// later — and anything that led with the third would bury the first.
|
|
func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|
inv := open.inventory
|
|
var out answers
|
|
var err error
|
|
|
|
out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.nodes, err = inv.Nodes(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
for _, n := range out.nodes {
|
|
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
|
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
|
// and could not.
|
|
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
|
out.quiet = append(out.quiet, n)
|
|
}
|
|
}
|
|
out.behind, err = inv.Behind(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
// And why any machine cannot be worked out at all, which is neither of the first two questions
|
|
// and is asked before them both in practice: a machine nothing can be computed for is not
|
|
// broken, not quiet and not behind, and every other answer here would call it well.
|
|
//
|
|
// Read through whoResolves, which is what the private network is built from, so this and the
|
|
// network agree about who could not be resolved rather than deciding it twice.
|
|
_, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
|
|
// And which machines are not running what the mesh would send them. The same question as a
|
|
// module being behind its source, one level down: that one says the catalogue is out of date,
|
|
// this one says a machine is — and only the second has anybody's change waiting in it.
|
|
//
|
|
// **One machine that cannot be resolved must not take the answer away from every other**
|
|
// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
|
|
// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
|
|
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
|
// reason is kept and reported as data; every question that does not depend on it is still
|
|
// answered.
|
|
would, err := wouldSend(ctx, open, out.nodes)
|
|
if err != nil {
|
|
out.network = err.Error()
|
|
would = map[string]string{}
|
|
}
|
|
out.waiting, err = inv.Waiting(ctx, would)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.reported, err = inv.LastReports(ctx)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.sources = map[string]inventory.Source{}
|
|
for module := range out.behind {
|
|
from, err := inv.SourceOf(ctx, module)
|
|
if err != nil {
|
|
return answers{}, err
|
|
}
|
|
out.sources[module] = from
|
|
}
|
|
return out, nil
|
|
}
|