461 lines
20 KiB
Go
461 lines
20 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"io"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// The things the mesh can be asked to do, separated from how it was asked.
|
|
//
|
|
// **A surface is an adapter with no decisions in it** (novox/hq ADR 0035). The command line and
|
|
// the command API call the same functions here, so an assignment refused at one is refused at the
|
|
// other for the same reason and in the same words. The moment a surface can accept something
|
|
// another would reject, the mesh has two answers to one question and people learn which to trust.
|
|
//
|
|
// Each returns what happened as text a person can read and a caller can pass on. Neither surface
|
|
// composes its own explanation, because two explanations of one refusal drift.
|
|
|
|
// assign puts a module on a node, and says at once what in the mesh no longer works out.
|
|
//
|
|
// The assignment is kept even when the node does not resolve: it is what a person meant, and
|
|
// assignment is not an ordering. A consumer assigned before its provider does not resolve for as
|
|
// long as it takes to assign the provider, and refusing the first half of a pair would make the
|
|
// order somebody types two commands in part of the mesh's rules.
|
|
//
|
|
// **What is checked is the mesh, not the one machine** — because that is what the assignment
|
|
// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own
|
|
// verify rejects, and it happens when the action's test is not the test the verify uses. Here the
|
|
// action tested one node and the verify is resolution over all of them, so an assignment could be
|
|
// reported as fine while it took a provision away from every other machine — a module offering a
|
|
// mesh-scoped provision stops offering it the moment its own node stops resolving, and every
|
|
// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a
|
|
// module already assigned. That was found on a four-node raise and read as a version bump breaking
|
|
// provider recognition; it was neither the version nor the provider.
|
|
//
|
|
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
|
// machines this just blocked is worth more than the milliseconds.
|
|
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
|
return assignWith(ctx, open, node, assignOptions{}, modules...)
|
|
}
|
|
|
|
// assignWith is assign asked with its options: build, for a module known and not built (novox/hq issue 325).
|
|
func assignWith(ctx context.Context, open *stores, node string, opts assignOptions, modules ...string) (string, error) {
|
|
if len(modules) == 0 {
|
|
return "", fmt.Errorf("assign %s names no module", node)
|
|
}
|
|
if err := refusedMovingTheController(modules); err != nil {
|
|
return "", err
|
|
}
|
|
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
|
// what the reads before it would say about a machine left out is not the answer's.
|
|
ctx = leftOutSaidBy(ctx, io.Discard)
|
|
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
|
// be taken without ever having been previewed (novox/hq ADR 0100).
|
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer release()
|
|
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
|
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
|
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
|
// are judged together (ADR 0207), so an act naming one not registered is not made in part.
|
|
if missing, err := notInCatalogue(ctx, open, modules); err != nil {
|
|
return "", err
|
|
} else if len(missing) > 0 {
|
|
if len(modules) == 1 {
|
|
return notRegistered(ctx, open, node, modules[0], opts)
|
|
}
|
|
var lines []string
|
|
for _, m := range missing {
|
|
where, err := whereIs(ctx, open.inventory, m, time.Now())
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
lines = append(lines, where.said)
|
|
}
|
|
return "", fmt.Errorf("%w: nothing was assigned, since modules assigned together are judged together "+
|
|
"(ADR 0207) and %s not registered:\n %s\n assign them together once each is registered, or each "+
|
|
"alone to keep it pending on its build", inventory.ErrNoSuchModule, strings.Join(missing, ", "),
|
|
strings.Join(lines, "\n "))
|
|
}
|
|
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
|
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
|
// resources are applied through a seat nothing on the node holds is refused, because that order
|
|
// — the service manager, the package manager and the runtime before anything that installs,
|
|
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
|
// holders that depend on each other go on in one command.
|
|
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
|
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
|
// assignment resolves against a record rather than against a coincidence.
|
|
settled, err := recordDerivedHolders(ctx, open)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var lines []string
|
|
var added []string
|
|
for _, module := range modules {
|
|
fresh, err := open.inventory.Assign(ctx, node, module)
|
|
if err != nil {
|
|
return strings.Join(lines, "\n"), err
|
|
}
|
|
if !fresh {
|
|
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
|
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
|
lines = append(lines, fmt.Sprintf(
|
|
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
|
continue
|
|
}
|
|
added = append(added, module)
|
|
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
|
}
|
|
if len(added) == 0 {
|
|
return strings.Join(lines, "\n"), nil
|
|
}
|
|
answer := strings.Join(lines, "\n")
|
|
for _, line := range settled {
|
|
answer += "\n " + line
|
|
}
|
|
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
|
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
|
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
|
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
|
var issued []string
|
|
for _, module := range added {
|
|
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
|
issued = append(issued, line)
|
|
}
|
|
}
|
|
// **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after
|
|
// the write: what this act changed about this node's unmet seat dependencies, and nothing else
|
|
// (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the
|
|
// rest of the node's list, and every other node's, is `status`'s — and whether the node resolves.
|
|
after := resolvedAfter(ctx, open, shelf, node)
|
|
for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) {
|
|
answer += "\n " + line
|
|
}
|
|
for _, line := range issued {
|
|
answer += "\n " + line
|
|
}
|
|
plan, err := after.plan, after.err
|
|
if err != nil {
|
|
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
|
// worth reporting: this machine's refusal is rarely the only consequence.
|
|
return answer + blockedElsewhere(ctx, open, node), err
|
|
}
|
|
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
|
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
|
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
|
isAdded := map[string]bool{}
|
|
for _, m := range added {
|
|
isAdded[m] = true
|
|
}
|
|
for _, u := range plan.Unhostable {
|
|
if !isAdded[u.Module] {
|
|
continue
|
|
}
|
|
for _, c := range u.Missing {
|
|
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
|
}
|
|
}
|
|
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
|
blockedElsewhere(ctx, open, node), nil
|
|
}
|
|
|
|
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
|
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
|
// and are not judged again.
|
|
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
|
map[string]catalogue.Manifest, []string, error) {
|
|
shelf, err := open.inventory.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
assigned, err := open.inventory.Assigned(ctx, node)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
already := map[string]bool{}
|
|
for _, a := range assigned {
|
|
already[a] = true
|
|
}
|
|
var adding []string
|
|
for _, m := range modules {
|
|
if !already[m] {
|
|
adding = append(adding, m)
|
|
}
|
|
}
|
|
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
|
// with everything else this act changed, so they are not said twice.
|
|
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
// Two modules declaring one path or unit, or one package present and absent, is refused before anything
|
|
// is recorded (novox/hq ADR 0210, ADR 0303, 04-ISSUES/235): kept, the node would not resolve until one came
|
|
// off again. Two modules declaring one package present is not a collision (ADR 0303, issue 628).
|
|
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return shelf, assigned, nil
|
|
}
|
|
|
|
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
|
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
|
//
|
|
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
|
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
|
// about the command's own output would ever have said so.
|
|
//
|
|
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
|
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
|
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
|
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
|
if err := refusedMovingTheController(modules); err != nil {
|
|
return "", err
|
|
}
|
|
if len(modules) == 0 {
|
|
return "", fmt.Errorf("unassign %s names no module", node)
|
|
}
|
|
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
|
ctx = leftOutSaidBy(ctx, io.Discard)
|
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer release()
|
|
shelf, err := open.inventory.Catalogue(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
assigned, err := open.inventory.Assigned(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
|
runs := map[string]bool{}
|
|
for _, a := range assigned {
|
|
runs[a] = true
|
|
}
|
|
// A module only pending on the node (novox/hq issue 325) is withdrawn, when it is the act's one module:
|
|
// that is the undo of an assignment kept while its build runs.
|
|
if len(modules) == 1 && !runs[modules[0]] {
|
|
said, withdrawn, err := withdrawPending(ctx, open.inventory, node, modules[0])
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if withdrawn {
|
|
return said, nil
|
|
}
|
|
}
|
|
for _, module := range modules {
|
|
if !runs[module] {
|
|
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
|
}
|
|
}
|
|
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
|
return "", err
|
|
}
|
|
for _, module := range modules {
|
|
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
|
node, strings.Join(modules, ", "), node)
|
|
after := resolvedAfter(ctx, open, shelf, node)
|
|
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
|
answer += "\n " + line
|
|
}
|
|
// Whether the node itself still resolves, from the same resolution: before, only the rest of the
|
|
// mesh's reads said it, on standard error and once per other machine.
|
|
if after.err != nil {
|
|
if unresolvable(after.err) {
|
|
answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node)
|
|
} else {
|
|
answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node)
|
|
}
|
|
for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") {
|
|
answer += "\n " + strings.TrimSpace(line)
|
|
}
|
|
}
|
|
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
|
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
|
answer += "\n " + line
|
|
}
|
|
return answer + blockedElsewhere(ctx, open, node), nil
|
|
}
|
|
|
|
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
|
// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet
|
|
// seat dependencies.
|
|
type afterAct struct {
|
|
plan catalogue.Resolution
|
|
err error
|
|
modules []string
|
|
unheld []catalogue.Unheld
|
|
}
|
|
|
|
// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves,
|
|
// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by
|
|
// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read
|
|
// after the write, which can only find more unmet than the closure would: a seat said held there is
|
|
// held in the closure too.
|
|
func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct {
|
|
plan, _, err := resolveNode(ctx, open, node)
|
|
if err == nil {
|
|
out := afterAct{plan: plan, unheld: plan.Unheld}
|
|
for _, m := range plan.Modules {
|
|
out.modules = append(out.modules, m.Module)
|
|
}
|
|
return out
|
|
}
|
|
out := afterAct{err: err}
|
|
assigned, readErr := open.inventory.Assigned(ctx, node)
|
|
if readErr != nil {
|
|
return out
|
|
}
|
|
out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned)
|
|
return out
|
|
}
|
|
|
|
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
|
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
|
// command API and the controller seat's verbs as they do from the command line.
|
|
func splitModules(field string) []string {
|
|
var out []string
|
|
for _, m := range strings.Split(field, ",") {
|
|
if m = strings.TrimSpace(m); m != "" {
|
|
out = append(out, m)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
|
//
|
|
// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the
|
|
// whole mesh twice and would still be a guess about which of several changes did it; saying
|
|
// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix,
|
|
// and cannot mislead. The machine that was just changed is left out because its own refusal is
|
|
// already the answer beside this one.
|
|
//
|
|
// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is
|
|
// not a reason to claim the assignment did not happen — it did.
|
|
func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
|
// One reading of the mesh for every machine this looks at.
|
|
ctx = withInputCache(ctx)
|
|
nodes, err := open.inventory.Nodes(ctx)
|
|
if err != nil {
|
|
return "\n\nThe rest of the mesh could not be checked: " + err.Error()
|
|
}
|
|
blocked := map[string]string{}
|
|
for _, n := range nodes {
|
|
if n.Name == except {
|
|
continue
|
|
}
|
|
if _, _, err := resolveNode(ctx, open, n.Name); err != nil {
|
|
blocked[n.Name] = err.Error()
|
|
}
|
|
}
|
|
if len(blocked) == 0 {
|
|
return ""
|
|
}
|
|
names := make([]string, 0, len(blocked))
|
|
for name := range blocked {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
|
|
var out strings.Builder
|
|
fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+
|
|
"nothing will be sent to them:\n", len(names))
|
|
for _, name := range names {
|
|
fmt.Fprintf(&out, " %s\n", name)
|
|
for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") {
|
|
fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line))
|
|
}
|
|
}
|
|
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
|
return out.String()
|
|
}
|
|
|
|
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
|
// says what it did in one line. Nothing for a module that declares no broker secret and runs no
|
|
// provisioner client; nothing for one whose user is already minted (a credential is rotated on
|
|
// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the
|
|
// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203).
|
|
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
|
inv := open.inventory
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
m, known := shelf[module]
|
|
if !known || mayIssue(m) != nil {
|
|
return ""
|
|
}
|
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
|
_, reads := m.OwnSecrets["broker"]
|
|
if !reads && !runsTheProvisionerClient(m) {
|
|
// A module whose bus accounts are its processes' alone has none of its own: only the processes that
|
|
// are not issued yet are issued (issue 1226).
|
|
return processAccountsOnAssign(ctx, inv, m, node)
|
|
}
|
|
if !reads {
|
|
// A module running the provisioner client alone is minted the client's account (issue 917).
|
|
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
|
}
|
|
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
|
return ""
|
|
} else if minted {
|
|
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
|
|
return processAccountsOnAssign(ctx, inv, m, node)
|
|
}
|
|
busAddress, err := broker.BusAddress()
|
|
if err == nil {
|
|
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
|
}
|
|
if err != nil {
|
|
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
|
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
|
}
|
|
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
|
}
|
|
|
|
// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did.
|
|
func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string {
|
|
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
|
if err != nil {
|
|
return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+
|
|
"`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module)
|
|
}
|
|
return line
|
|
}
|
|
|
|
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
|
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
|
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
|
// change that says who may change what.
|
|
func refusedMovingTheController(modules []string) error {
|
|
verb, through := throughAVerb()
|
|
if !through || !slices.Contains(modules, controllerModule) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+
|
|
"never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+
|
|
"terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb)
|
|
}
|