427 lines
16 KiB
Go
427 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// working out what one machine should be.
|
|
//
|
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
|
|
|
// notResolvable marks the one failure in planFor that is a statement about the node: its assigned
|
|
// modules do not compose. Every other failure means the mesh could not be *asked* — the store was
|
|
// unreachable, a key could not be read — and says nothing about the node at all.
|
|
//
|
|
// The distinction exists because three callers gather something across every machine and must carry
|
|
// on when one machine's set is broken. Each of them read a plain error as "their set does not
|
|
// resolve", and so read a store that was briefly unreachable as a machine that runs nothing. On the
|
|
// roster of routed names that is not a degraded answer but a false one: it states, to every machine
|
|
// at once, that another machine's names do not exist. A control node spent hours replacing every
|
|
// container it ran, on a six-minute cycle, because each pass restarted the store this is read from,
|
|
// the read failed, one name left the roster, and the roster is part of every container's identity
|
|
// (novox/hq 04-ISSUES/152, and 04-ISSUES/151 for why a changed roster is a changed container).
|
|
//
|
|
// So: skip a node that cannot resolve, and never a node that could not be read.
|
|
type notResolvable struct{ err error }
|
|
|
|
func (n notResolvable) Error() string { return n.err.Error() }
|
|
func (n notResolvable) Unwrap() error { return n.err }
|
|
|
|
// unresolvable reports whether err is a node's own set failing to compose, rather than the mesh
|
|
// being unable to answer.
|
|
func unresolvable(err error) bool {
|
|
var n notResolvable
|
|
return errors.As(err, &n)
|
|
}
|
|
|
|
// leftOutKey carries where theRestOfTheMesh says a machine it leaves out, for an act that answers with
|
|
// its own resolution instead (novox/hq issue 237).
|
|
type leftOutKey struct{}
|
|
|
|
// leftOutSaidBy is ctx with theRestOfTheMesh's "left out" lines written to w rather than to standard
|
|
// error.
|
|
//
|
|
// **An act's answer is one resolution taken after its write** (novox/hq issue 237, decided 2026-10-11).
|
|
// `assign` read the whole mesh once before its write — to record the seats held by derivation — and that
|
|
// read said, on standard error, that the node did not resolve for lack of the seat the act was about to
|
|
// give it. The controller seat's verb answers with both streams, so the stale line came last, after the
|
|
// act's own "now has it held", and named as its remedy the assignment just made. An act gives io.Discard
|
|
// here: what the node resolves to after the write is said by its own planFor, and every other machine
|
|
// that does not resolve by blockedElsewhere, both read after the write.
|
|
func leftOutSaidBy(ctx context.Context, w io.Writer) context.Context {
|
|
return context.WithValue(ctx, leftOutKey{}, w)
|
|
}
|
|
|
|
// declarationFor is everything a node would be sent.
|
|
//
|
|
// One place, because there were three and one of them was written before credentials existed and
|
|
// silently produced a declaration missing them — a difference between what `plan` showed and what
|
|
// `plan --json` handed to anything reading it.
|
|
func declarationFor(ctx context.Context, open *stores, node string,
|
|
plan catalogue.Resolution, settings catalogue.SettingsBy) (sendable, error) {
|
|
// **A send, because `plan` is the send without the sending.** It is one machine, named by
|
|
// a person, who is asking what a push would do — so the port it shows and the secret it seals
|
|
// have to be the ones a push would use, and both are kept once chosen. Showing numbers that a
|
|
// later push would replace would make the command answer a question nobody asked.
|
|
//
|
|
// The line is not "a question may not write". It is who is asking and how often: this is a
|
|
// person, about one machine, on purpose. What may not write is the comparison the mesh runs
|
|
// over every machine to answer whether each is up to date, which is a question and makes nothing (composeQuestion).
|
|
return composeSend(ctx, open, node)
|
|
}
|
|
|
|
// boundToData is every binding of this machine's consumers to a provision that keeps their data
|
|
// (novox/hq ADR 0232), as a send records it. Not a consumer left out of the declaration: the machine
|
|
// is not told anything new about it, so nothing about where it is bound has been sent.
|
|
func boundToData(plan catalogue.Resolution, leftOut map[string]string) []inventory.Binding {
|
|
var out []inventory.Binding
|
|
seen := map[[2]string]bool{}
|
|
for _, n := range plan.Needs {
|
|
if !n.KeepsData || n.ByRecord || n.Module == "" {
|
|
continue
|
|
}
|
|
if _, left := leftOut[n.For]; left {
|
|
continue
|
|
}
|
|
key := [2]string{n.For, n.Name}
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
out = append(out, inventory.Binding{Machine: plan.Node, Consumer: n.For, Provision: n.Name,
|
|
Provider: catalogue.Chosen{Node: n.From, Module: n.Module}})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
|
// for a reordering nobody made.
|
|
func sortedKeysOf(m map[string]string) []string {
|
|
if len(m) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
|
|
// the process of the module running it runs as, empty where no such module is on the machine.
|
|
func provisionerAccount(plan catalogue.Resolution) string {
|
|
for _, m := range plan.Modules {
|
|
if !runsTheProvisionerClient(m) {
|
|
continue
|
|
}
|
|
for _, r := range m.Resources {
|
|
if r["type"] == "process" {
|
|
if user, _ := r["user"].(string); user != "" && user != "root" {
|
|
return user
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// certificateFor is what the mesh certifies about one machine's internal name.
|
|
//
|
|
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
|
|
// the machine and whether it is on the private network, `identity` holds the authority and the
|
|
// key that machine reported. The process holding both grants asks each for its part
|
|
// (novox/hq ADR 0008).
|
|
func certificateFor(ctx context.Context, open *stores, node string) (string, string, error) {
|
|
inv := open.inventory
|
|
ident, err := open.Identity(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
serving, err := ident.ServingKeyOf(ctx, record.ID)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if serving == "" {
|
|
// The machine joined before it had one, or never reported it. Said plainly, because the
|
|
// remedy is on the machine and no amount of pushing from here will produce one.
|
|
return "", "", fmt.Errorf(
|
|
"%s wants a certificate and has never told the mesh what key it serves with; it "+
|
|
"joins again to report one", node)
|
|
}
|
|
|
|
// The name it is certified for. Only a machine on the private network has one — a certificate
|
|
// for a name nothing resolves is a certificate nothing can check.
|
|
//
|
|
// With the catalogue, not without it. Being on the private network is a conclusion about what
|
|
// a node resolves to, so a nil shelf resolves nothing and every machine looks like it is on no
|
|
// network — which refused every certificate the mesh was asked for, and said the machine was
|
|
// not on a network it plainly was.
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
where, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
name := where[node]
|
|
if name == "" {
|
|
return "", "", fmt.Errorf(
|
|
"%s wants a certificate and is not on the private network, so it has no name inside "+
|
|
"the mesh to be certified for", node)
|
|
}
|
|
|
|
issued, err := ident.Certify(ctx, node, name, serving)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
authority, err := ident.EstablishAuthority(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
return issued, authority.Certificate, nil
|
|
}
|
|
|
|
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
|
|
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
|
|
// assigned over the manifest's own, settled with the node's settings layers.
|
|
//
|
|
// **The one derivation** for every reader of a provider's address — a consumer's binding, the
|
|
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
|
// given ports are that node's refusal to report, not this reader's.
|
|
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
|
m catalogue.Manifest, provision string) (map[string]any, []catalogue.Layer, error) {
|
|
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
serves := catalogue.ServedOn(m, provision, ports)
|
|
if len(serves) > 0 {
|
|
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
}
|
|
// The layers too, read once, for what is filled after (novox/hq ADR 0346).
|
|
return serves, layers, nil
|
|
}
|
|
|
|
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
|
// node's settings layers for the module, read once for both.
|
|
func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
|
|
m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) {
|
|
ports, err := portsOn(ctx, inv, node, m.Module)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
|
for wanted, at := range given {
|
|
ports[wanted] = at
|
|
}
|
|
}
|
|
return ports, layers, nil
|
|
}
|
|
|
|
// portsOn is one module's assignments on one machine, by the port the software uses.
|
|
func portsOn(
|
|
ctx context.Context, inv *inventory.Inventory, node, module string,
|
|
) (map[int]int, error) {
|
|
all, err := inv.PortsFor(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[int]int{}
|
|
for _, a := range all {
|
|
if a.Module == module {
|
|
out[a.Wanted] = a.Machine
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
|
|
// for want of a credential. Asked on every send to decide whether the machine holding the bus must
|
|
// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
|
|
func busUserList(ctx context.Context, inv *inventory.Inventory,
|
|
onThisNode []catalogue.Manifest) (string, []string, error) {
|
|
holdsTheBus := false
|
|
for _, m := range onThisNode {
|
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
|
holdsTheBus = true
|
|
}
|
|
}
|
|
if !holdsTheBus {
|
|
return "", nil, nil
|
|
}
|
|
|
|
records, err := inv.BusRecords(ctx)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
users, err := broker.Users(records)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
kept, err := inv.BusUsers(ctx)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
hashes := make(map[string]string, len(kept))
|
|
for name, u := range kept {
|
|
hashes[name] = u.PasswordHash
|
|
}
|
|
filled, missing := broker.WithPasswords(users, hashes)
|
|
if len(filled) == 0 {
|
|
return "", missing, fmt.Errorf(
|
|
"this machine runs the bus and not one user has a credential, so the composed list " +
|
|
"would refuse every connection in the mesh")
|
|
}
|
|
list, err := broker.ComposeAccounts(filled)
|
|
return list, missing, err
|
|
}
|
|
|
|
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
|
// own set when the provider is here, else the one the catalogue says offers it.
|
|
// issuedOutsideBy is whether a need's provision is offered by its provider with a credential issued outside the
|
|
// mesh (novox/hq ADR 0337), and that provider module: the one answering it when known, else any that offers it so.
|
|
func issuedOutsideBy(shelf map[string]catalogue.Manifest, n catalogue.Needed) (string, bool) {
|
|
if n.Module != "" {
|
|
if m, known := shelf[n.Module]; known {
|
|
return n.Module, m.IssuedOutsideOf(n.Name)
|
|
}
|
|
}
|
|
return catalogue.OffersIssuedOutside(shelf, n.Name)
|
|
}
|
|
|
|
// unheldLogger remembers what was last logged about each node's unmet seat dependencies, so the
|
|
// log says each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
|
// A test gives its own writer and memory, and no longer shares the process's (novox/hq issue 486).
|
|
//
|
|
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
|
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
|
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
|
// burying the line about the node it acted on. A command says what concerns its own act instead
|
|
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
|
type unheldLogger struct {
|
|
mu sync.Mutex
|
|
logged map[string]string
|
|
out io.Writer
|
|
}
|
|
|
|
func newUnheldLogger(out io.Writer) *unheldLogger {
|
|
return &unheldLogger{logged: map[string]string{}, out: out}
|
|
}
|
|
|
|
var (
|
|
unheldLog = newUnheldLogger(os.Stderr)
|
|
logUnheldChanges bool
|
|
)
|
|
|
|
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
|
|
// it, including when they become none — in the serving controller, and nowhere else.
|
|
func logUnheld(node string, unheld []catalogue.Unheld) {
|
|
if !logUnheldChanges {
|
|
return
|
|
}
|
|
unheldLog.log(node, unheld)
|
|
}
|
|
|
|
func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) {
|
|
lines := make([]string, 0, len(unheld))
|
|
for _, u := range unheld {
|
|
lines = append(lines, u.String())
|
|
}
|
|
now := strings.Join(lines, "\n")
|
|
l.mu.Lock()
|
|
defer l.mu.Unlock()
|
|
before, seen := l.logged[node]
|
|
l.logged[node] = now
|
|
if (seen && before == now) || (!seen && now == "") {
|
|
return
|
|
}
|
|
if now == "" {
|
|
fmt.Fprintf(l.out, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
|
return
|
|
}
|
|
fmt.Fprintf(l.out, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
|
node, len(lines), strings.Join(lines, "\n "))
|
|
}
|
|
|
|
// unheldOver is the node's unmet seat dependencies judged over a set of module names alone.
|
|
func unheldOver(shelf map[string]catalogue.Manifest, node string, names []string) []catalogue.Unheld {
|
|
var set []catalogue.Manifest
|
|
for _, n := range names {
|
|
if m, known := shelf[n]; known {
|
|
set = append(set, m)
|
|
}
|
|
}
|
|
return catalogue.UnheldDependencies(shelf, node, set, nil)
|
|
}
|
|
|
|
// unheldChangeTo is unheldChange with the node's dependencies as they now stand given: `now` is what
|
|
// the act's one resolution after its write says is unmet, and `after` the modules that resolution has
|
|
// on the node (novox/hq issue 237). A line about how things stand after the act is never judged from
|
|
// anything else, so it cannot disagree with `plan` for the node.
|
|
func unheldChangeTo(shelf map[string]catalogue.Manifest, node string, before, after []string,
|
|
now []catalogue.Unheld) []string {
|
|
key := func(u catalogue.Unheld) string { return u.Module + " " + u.Seat }
|
|
was := map[string]catalogue.Unheld{}
|
|
for _, u := range unheldOver(shelf, node, before) {
|
|
was[key(u)] = u
|
|
}
|
|
is := map[string]catalogue.Unheld{}
|
|
for _, u := range now {
|
|
is[key(u)] = u
|
|
}
|
|
var lines []string
|
|
for _, k := range sortedNames(is) {
|
|
if _, already := was[k]; !already {
|
|
lines = append(lines, "but "+is[k].String())
|
|
}
|
|
}
|
|
for _, k := range sortedNames(was) {
|
|
if _, still := is[k]; still {
|
|
continue
|
|
}
|
|
u := was[k]
|
|
if !slices.Contains(after, u.Module) {
|
|
continue // went with its module, which says nothing about the seat
|
|
}
|
|
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
|
|
}
|
|
return lines
|
|
}
|
|
|
|
func sortedNames[V any](m map[string]V) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|