jschoubben
1c117d1189
A keep wait on a filesystem that cannot carry ACLs is not excused, and nothing is asked; one whose filesystem must first enable them says in the ask that the take sets it. The tool runner's account is sent with no uid, for the engine to resolve by name.
365 lines
15 KiB
Go
365 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
|
//
|
|
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
|
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
|
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
|
//
|
|
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
|
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
|
// machine;
|
|
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
|
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
|
//
|
|
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
|
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
|
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
|
// why, and raises the module's `unhealthy` condition.
|
|
|
|
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
|
const kindNeedsOperator = "needs-operator"
|
|
|
|
// needsOperatorKey is a module's needs-operator condition on a machine.
|
|
func needsOperatorKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
|
}
|
|
|
|
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
|
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
|
// and no wait of it is excused.
|
|
type operatorWaitFacts struct {
|
|
manifests map[string]catalogue.Manifest
|
|
given map[string]map[string]time.Time
|
|
// placed is, per "<module>@<machine>", where each access of the module is on that machine, by id (novox/hq ADR
|
|
// 0325 §7); taken is, per machine, every folder the operator let the mesh keep there, with the filesystem it was
|
|
// taken on (empty when the engine named none). A machine absent from taken could not be read, and no access wait
|
|
// of it is excused.
|
|
placed map[string]map[string]string
|
|
taken map[string]map[string]string
|
|
}
|
|
|
|
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
|
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
|
m, known := f.manifests[module]
|
|
if !known {
|
|
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
|
}
|
|
named := 0
|
|
for _, n := range []string{w.Secret, w.Setting, w.Access} {
|
|
if n != "" {
|
|
named++
|
|
}
|
|
}
|
|
switch {
|
|
case named != 1:
|
|
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting, or one access", waitNames(w))
|
|
case w.Access != "":
|
|
return checkAccessWait(module, machine, m, w, f)
|
|
case w.Setting != "":
|
|
if _, declared := m.Settings[w.Setting]; !declared {
|
|
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
|
}
|
|
return nil
|
|
}
|
|
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
|
if !declared {
|
|
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
|
}
|
|
if own.IssuedBy != catalogue.IssuedOutside {
|
|
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
|
"the mesh waits for the operator", w.Secret)
|
|
}
|
|
given, readable := f.given[module+"@"+machine]
|
|
if !readable {
|
|
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
|
}
|
|
if at, was := given[w.Secret]; was {
|
|
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
|
at.UTC().Format("2006-01-02 15:04 MST"))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// checkAccessWait is nil when an access wait is excused (novox/hq ADR 0325 §7): the module's manifest declares the
|
|
// access, its folder is placed on that machine, and either the folder is absent and its disk is not there (§9), or
|
|
// it has not yet been taken under the operator's warrant (§8). That a folder that is there is not owned by root or a
|
|
// root group is the node-engine's to see: it states such an access unhealthy and never waiting (§12). Any other
|
|
// access wait is judged unhealthy, saying why: a module that gains an access to a folder already taken is given its
|
|
// entries without a wait.
|
|
func checkAccessWait(module, machine string, m catalogue.Manifest, w inventory.Wait, f operatorWaitFacts) error {
|
|
declared := false
|
|
for _, a := range m.Accesses {
|
|
declared = declared || a.ID == w.Access
|
|
}
|
|
if !declared {
|
|
return fmt.Errorf("says it waits for the access %s, which %s does not declare", w.Access, module)
|
|
}
|
|
path, placed := f.placed[module+"@"+machine][w.Access]
|
|
if !placed {
|
|
return fmt.Errorf("says it waits for the access %s, which is not placed on %s", w.Access, machine)
|
|
}
|
|
switch {
|
|
case w.Reason == link.WaitDisk || (w.Reason == "" && w.Disk):
|
|
return nil
|
|
case w.Reason != link.WaitKeep && w.Reason != "":
|
|
return fmt.Errorf("says it waits for the access %s for %q, and an access waits for the operator's word to "+
|
|
"keep its folder or for its disk", w.Access, w.Reason)
|
|
case w.ACL == link.ACLUnsupported:
|
|
// No act can make this filesystem carry access entries: nothing to ask; the operator places the data elsewhere.
|
|
return fmt.Errorf("says it waits for the access %s, whose folder's filesystem cannot carry access entries: "+
|
|
"the operator places the data on one that can, or mounts it with ACLs", w.Access)
|
|
}
|
|
taken, readable := f.taken[machine]
|
|
if !readable {
|
|
return fmt.Errorf("says it waits for the access %s, and which folders are taken on %s could not be read",
|
|
w.Access, machine)
|
|
}
|
|
// A folder taken on another filesystem than the one it is on now (it moved to another disk) is not taken by that
|
|
// warrant: the wait is excused, and the operator is asked again (ADR 0325 §8).
|
|
if on, was := taken[path]; was && (w.Placed == "" || on == w.Placed) {
|
|
return fmt.Errorf("says it waits for the access %s, whose folder is taken on %s already: a module that gains "+
|
|
"an access to a taken folder is given its entries without a wait", w.Access, machine)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
|
const waitRefusedPrefix = "says it waits"
|
|
|
|
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
|
func waitNames(w inventory.Wait) string {
|
|
switch {
|
|
case w.Secret != "" && w.Setting != "":
|
|
return "the secret " + w.Secret + " and the setting " + w.Setting
|
|
case w.Access != "" && (w.Secret != "" || w.Setting != ""):
|
|
return "the access " + w.Access + " and a secret or a setting"
|
|
case w.Access != "":
|
|
return "the access " + w.Access
|
|
case w.Secret != "":
|
|
return "the secret " + w.Secret
|
|
case w.Setting != "":
|
|
return "the setting " + w.Setting
|
|
}
|
|
return "nothing"
|
|
}
|
|
|
|
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
|
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
|
// statement as kept is not changed.
|
|
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
|
out := make([]inventory.ResourceHealth, 0, len(rs))
|
|
for _, r := range rs {
|
|
if r.State == link.StateWaiting {
|
|
var why error
|
|
if len(r.Waits) == 0 {
|
|
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
|
}
|
|
for _, w := range r.Waits {
|
|
if why == nil {
|
|
why = checkWait(r.Module, machine, w, f)
|
|
}
|
|
}
|
|
if why != nil {
|
|
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
|
}
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
|
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
|
// does not wait: it is judged as before.
|
|
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
|
var waits []inventory.Wait
|
|
for _, r := range rs {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateWaiting:
|
|
waits = append(waits, r.Waits...)
|
|
default:
|
|
return nil, false
|
|
}
|
|
}
|
|
return waits, len(waits) > 0
|
|
}
|
|
|
|
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
|
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
|
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
|
var parts []string
|
|
for _, w := range waits {
|
|
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
|
// An access in the controller's own words, never the engine's: words with no path (novox/hq ADR 0325 §7).
|
|
switch {
|
|
case w.Access != "" && w.Disk:
|
|
part = fmt.Sprintf("the disk that holds %s mounted (%s", w.Access, waitNames(w))
|
|
case w.Access != "":
|
|
part = fmt.Sprintf("leave to keep the shared folder %s (%s, asked of you once for the folder", w.Access,
|
|
waitNames(w))
|
|
}
|
|
if w.Secret != "" {
|
|
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
|
}
|
|
parts = append(parts, part+")")
|
|
}
|
|
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
|
}
|
|
|
|
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
|
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
|
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
|
o.Summary = operatorWaitSaid(module, node, waits)
|
|
w := needsOperatorWords(module, node, waits)
|
|
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
|
return o
|
|
}
|
|
|
|
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
|
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
|
// the setting's names, and the line, are in the summary for whoever looks closer.
|
|
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
|
var acts []string
|
|
seen := map[string]bool{}
|
|
secret := false
|
|
for _, w := range waits {
|
|
var act string
|
|
switch {
|
|
case w.Secret != "":
|
|
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
|
case w.Setting != "":
|
|
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
|
// An access, in words with no path (novox/hq ADR 0325 §7): the folder by the access's name.
|
|
case w.Access != "" && w.Disk:
|
|
act = fmt.Sprintf("mount the disk that holds %s on %s", w.Access, node)
|
|
case w.Access != "":
|
|
act = fmt.Sprintf("let the mesh keep the shared folder %s on %s when you are asked", w.Access, node)
|
|
}
|
|
if act != "" && !seen[act] {
|
|
seen[act] = true
|
|
acts = append(acts, act)
|
|
}
|
|
}
|
|
needs := strings.Join(acts, "; and ") + "."
|
|
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
|
// summary instead.
|
|
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
|
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
|
}
|
|
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
|
if secret {
|
|
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
|
"is sealed to the machine."
|
|
}
|
|
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
|
return words{
|
|
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
|
Needs: needs,
|
|
Explanation: explanation,
|
|
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
|
}
|
|
}
|
|
|
|
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
|
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
|
// unknown, so none of its waits is excused.
|
|
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
|
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
|
if f.manifests == nil {
|
|
f.manifests = map[string]catalogue.Manifest{}
|
|
}
|
|
if f.given == nil {
|
|
f.given = map[string]map[string]time.Time{}
|
|
}
|
|
if f.placed == nil {
|
|
f.placed = map[string]map[string]string{}
|
|
}
|
|
if f.taken == nil {
|
|
f.taken = map[string]map[string]string{}
|
|
}
|
|
if inv == nil {
|
|
return
|
|
}
|
|
var shelf map[string]catalogue.Manifest
|
|
sort.Strings(modules)
|
|
for _, module := range modules {
|
|
if _, has := f.manifests[module]; !has {
|
|
if m, given := manifests[module]; given {
|
|
f.manifests[module] = m
|
|
} else {
|
|
if shelf == nil {
|
|
var err error
|
|
if shelf, err = inv.Catalogue(ctx); err != nil {
|
|
shelf = map[string]catalogue.Manifest{}
|
|
}
|
|
}
|
|
if m, known := shelf[module]; known {
|
|
f.manifests[module] = m
|
|
}
|
|
}
|
|
}
|
|
// Where each access of it is placed there (ADR 0325 §7): read once per module and machine.
|
|
if m, known := f.manifests[module]; known && len(m.Accesses) > 0 {
|
|
if _, read := f.placed[module+"@"+machine]; !read {
|
|
if layers, err := inv.SettingsFor(ctx, machine, module); err == nil {
|
|
placed := map[string]string{}
|
|
for _, a := range m.Accesses {
|
|
if path, has := catalogue.AccessPathOn(m, layers, a.ID); has && a.ID != "" {
|
|
placed[a.ID] = path
|
|
}
|
|
}
|
|
f.placed[module+"@"+machine] = placed
|
|
}
|
|
}
|
|
}
|
|
if _, read := f.given[module+"@"+machine]; read {
|
|
continue
|
|
}
|
|
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
|
f.given[module+"@"+machine] = given
|
|
}
|
|
}
|
|
if _, read := f.taken[machine]; !read {
|
|
if taken, err := inv.KeptFolders(ctx, machine); err == nil {
|
|
kept := map[string]string{}
|
|
for path, t := range taken {
|
|
kept[path] = t.Placed
|
|
}
|
|
f.taken[machine] = kept
|
|
}
|
|
}
|
|
}
|
|
|
|
// waitingModules is every module with a waiting resource in a statement.
|
|
func waitingModules(rs []inventory.ResourceHealth) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range rs {
|
|
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
|
seen[r.Module] = true
|
|
out = append(out, r.Module)
|
|
}
|
|
}
|
|
return out
|
|
}
|