jochen
c9dd07f919
mesh/merge-gate head c9dd07f919a2 merged into trunk 404b4409e70e — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head c9dd07f919a2 merged into trunk 404b4409e70e — pass: its merge-check.sh passed (run 2)
mesh/delivery delivered
A Servarr app's API key, an indexer's key and a downloader's password are values the provider's
software holds; the pair path made a random value for every consumer of them, sealed it to both ends
and reported success, and the provider refused it. An offer may now say `"credential": {"issued-by":
"outside"}`: SecretFor never makes a value for it, the plan reads such a pair as given or not given
and sends the consumer no credential file until a person gives it, without refusing its machine;
remakeShared and SharedSecretFor never make a provider's own secret issued outside either.
`secret accept <provider's node> <provider module> <provision>` seals one value to every consumer that
binds the provision from that provider, the provider and the operator, under one generation; the
per-pair form is refused for such a provision, and rotate names the accept. A consumer may wait for a
provision's credential: the controller excuses the wait only when the store bears it out, and its
needs-operator condition names the one accept line.
Adds the replay of issue 164 (TestReplay164), written with only what the controller had before.
387 lines
15 KiB
Go
387 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).
|
|
//
|
|
// **A module's tool check may say it waits**: nothing of it is wrong but a part that cannot work until the operator
|
|
// gives one of its own secrets or one of its settings. The node-engine states such a resource `waiting`, with what
|
|
// it waits for. A module saying so is an assertion, so **the controller checks each wait before it excuses it**:
|
|
//
|
|
// - a wait for a secret names an own secret the module's manifest declares — by its name, or as a member of a
|
|
// secret family — said `"issued-by": "outside"`, and the store holds no value a person gave for it on that
|
|
// machine;
|
|
// - a wait for a setting names a setting the manifest declares (checked by name only: the controller cannot tell
|
|
// whether a free-form value covers a part, and the needs-operator condition is where a false one shows).
|
|
//
|
|
// An excused wait is read by the first-node gate as *waits for a person* (ADR 0254), a pass carried in the verdict,
|
|
// for any build of the module — a secret not given is owed by every build alike. It is said to the operator as
|
|
// `module.<module>.<machine>.needs-operator`, naming the act. A wait that fails the check is judged unhealthy, saying
|
|
// why, and raises the module's `unhealthy` condition.
|
|
|
|
// kindNeedsOperator is a module's condition while a part of it waits for the operator's secret or setting.
|
|
const kindNeedsOperator = "needs-operator"
|
|
|
|
// needsOperatorKey is a module's needs-operator condition on a machine.
|
|
func needsOperatorKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindNeedsOperator)
|
|
}
|
|
|
|
// operatorWaitFacts is what the controller holds to check a module's waits: the manifest judged per module, and per
|
|
// "<module>@<machine>" the own secrets a person gave there, with when. A module or a machine absent is not known,
|
|
// and no wait of it is excused.
|
|
type operatorWaitFacts struct {
|
|
manifests map[string]catalogue.Manifest
|
|
given map[string]map[string]time.Time
|
|
// outside is, per provision some module offers with a credential issued outside the mesh, that provider module
|
|
// and the machines it is assigned to (novox/hq ADR 0337); pairsGiven, per "<module>@<machine>", the provisions
|
|
// it holds a value a person gave for, sealed to the keys both ends have now. Absent is not known: no wait for a
|
|
// provision of it is excused.
|
|
outside map[string]outsideProvider
|
|
pairsGiven map[string]map[string]bool
|
|
}
|
|
|
|
// outsideProvider is the module offering a provision whose credential only a person gives, and where it runs.
|
|
type outsideProvider struct {
|
|
Module string
|
|
Machines []string
|
|
}
|
|
|
|
// acceptLine is the one line that gives an outside provision's credential for every consumer of its provider: on
|
|
// the provider's machine when it runs on one, named as <machine> when on several or none.
|
|
func (p outsideProvider) acceptLine(provision string) string {
|
|
machine := "<its machine>"
|
|
if len(p.Machines) == 1 {
|
|
machine = p.Machines[0]
|
|
}
|
|
return inventory.AcceptLine(machine, p.Module, provision)
|
|
}
|
|
|
|
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
|
func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) error {
|
|
m, known := f.manifests[module]
|
|
if !known {
|
|
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
|
}
|
|
named := 0
|
|
for _, n := range []string{w.Secret, w.Setting, w.Provision} {
|
|
if n != "" {
|
|
named++
|
|
}
|
|
}
|
|
switch {
|
|
case named != 1:
|
|
return fmt.Errorf("says it waits, naming %s, where a wait names one secret, one setting or one provision",
|
|
waitNames(w))
|
|
case w.Provision != "":
|
|
return checkProvisionWait(module, machine, w, m, f)
|
|
case w.Setting != "":
|
|
if _, declared := m.Settings[w.Setting]; !declared {
|
|
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
|
}
|
|
return nil
|
|
}
|
|
own, _, declared := m.OwnSecrets.Lookup(w.Secret)
|
|
if !declared {
|
|
return fmt.Errorf("says it waits for the secret %s, which %s does not declare", w.Secret, module)
|
|
}
|
|
if own.IssuedBy != catalogue.IssuedOutside {
|
|
return fmt.Errorf("says it waits for the secret %s, which the mesh makes itself: only a secret issued outside "+
|
|
"the mesh waits for the operator", w.Secret)
|
|
}
|
|
given, readable := f.given[module+"@"+machine]
|
|
if !readable {
|
|
return fmt.Errorf("says it waits for the secret %s, and what was given on %s could not be read", w.Secret, machine)
|
|
}
|
|
if at, was := given[w.Secret]; was {
|
|
return fmt.Errorf("says it waits for the secret %s, which was given at %s", w.Secret,
|
|
at.UTC().Format("2006-01-02 15:04 MST"))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// checkProvisionWait excuses a wait for a provision's credential only when the store bears it out (novox/hq ADR 0337
|
|
// §6): the module requires the provision, a provider offers it with a credential issued outside the mesh, and the
|
|
// module on this machine holds no value a person gave for it under the keys both ends have now.
|
|
func checkProvisionWait(module, machine string, w inventory.Wait, m catalogue.Manifest, f operatorWaitFacts) error {
|
|
if !slices.Contains(m.Requires, w.Provision) {
|
|
return fmt.Errorf("says it waits for the provision %s, which %s does not require", w.Provision, module)
|
|
}
|
|
if _, outside := f.outside[w.Provision]; !outside {
|
|
return fmt.Errorf("says it waits for the provision %s, whose credential the mesh gives itself: only a "+
|
|
"credential issued outside the mesh waits for the operator", w.Provision)
|
|
}
|
|
given, readable := f.pairsGiven[module+"@"+machine]
|
|
if !readable {
|
|
return fmt.Errorf("says it waits for the provision %s, and what was given on %s could not be read", w.Provision, machine)
|
|
}
|
|
if given[w.Provision] {
|
|
return fmt.Errorf("says it waits for the provision %s, whose credential was given", w.Provision)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
|
const waitRefusedPrefix = "says it waits"
|
|
|
|
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
|
func waitNames(w inventory.Wait) string {
|
|
var names []string
|
|
if w.Secret != "" {
|
|
names = append(names, "the secret "+w.Secret)
|
|
}
|
|
if w.Setting != "" {
|
|
names = append(names, "the setting "+w.Setting)
|
|
}
|
|
if w.Provision != "" {
|
|
names = append(names, "the provision "+w.Provision)
|
|
}
|
|
if len(names) == 0 {
|
|
return "nothing"
|
|
}
|
|
return strings.Join(names, " and ")
|
|
}
|
|
|
|
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
|
// kept as said; one with a wait that does not, or with no wait at all, is answered as unhealthy with why. Pure; the
|
|
// statement as kept is not changed.
|
|
func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitFacts) []inventory.ResourceHealth {
|
|
out := make([]inventory.ResourceHealth, 0, len(rs))
|
|
for _, r := range rs {
|
|
if r.State == link.StateWaiting {
|
|
var why error
|
|
if len(r.Waits) == 0 {
|
|
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
|
}
|
|
waits := make([]inventory.Wait, 0, len(r.Waits))
|
|
for _, w := range r.Waits {
|
|
if why == nil {
|
|
why = checkWait(r.Module, machine, w, f)
|
|
}
|
|
if p, outside := f.outside[w.Provision]; outside && w.Provision != "" {
|
|
w.Act = p.acceptLine(w.Provision)
|
|
}
|
|
waits = append(waits, w)
|
|
}
|
|
r.Waits = waits
|
|
if why != nil {
|
|
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
|
}
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// operatorWait is whether everything not healthy of a module on a machine is waiting with its waits checked
|
|
// (checkWaiting already applied), and those waits. A module with anything unhealthy, starting or unknown beside it
|
|
// does not wait: it is judged as before.
|
|
func operatorWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, bool) {
|
|
var waits []inventory.Wait
|
|
for _, r := range rs {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateWaiting:
|
|
waits = append(waits, r.Waits...)
|
|
default:
|
|
return nil, false
|
|
}
|
|
}
|
|
return waits, len(waits) > 0
|
|
}
|
|
|
|
// operatorWaitSaid is a module's wait for the operator in one sentence, for the gate's verdict and the condition's
|
|
// summary: what the operator gives and what it names, and for a secret the line that opens the desk prompt.
|
|
func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
|
var parts []string
|
|
for _, w := range waits {
|
|
part := fmt.Sprintf("%s (%s", w.What, waitNames(w))
|
|
if w.Secret != "" {
|
|
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
|
}
|
|
if w.Provision != "" && w.Act != "" {
|
|
// One line for the provider, which seals it to every consumer of it (ADR 0337 §3, §7).
|
|
part += fmt.Sprintf(", given for every consumer with `%s`", w.Act)
|
|
}
|
|
parts = append(parts, part+")")
|
|
}
|
|
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
|
}
|
|
|
|
// needsOperatorObservation is a module whose only parts not healthy wait for the operator (ADR 0283): the operator's,
|
|
// a warning however long it stands, its plain words naming the act and never saying there is nothing to do.
|
|
func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Severity = kindNeedsOperator, kindNeedsOperator, conditions.ResolverOperator, conditions.Warning
|
|
o.Summary = operatorWaitSaid(module, node, waits)
|
|
w := needsOperatorWords(module, node, waits)
|
|
o.Headline, o.Explanation, o.Needs, o.Resolved, o.Actions = w.Headline, w.Explanation, w.Needs, w.Resolved, nil
|
|
return o
|
|
}
|
|
|
|
// needsOperatorWords is what the operator reads of a module waiting for them (ADR 0253, ADR 0283): the act, for a
|
|
// secret typed at the machine's desk prompt and for a setting approved when an agent proposes it. The secret's and
|
|
// the setting's names, and the line, are in the summary for whoever looks closer.
|
|
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
|
var acts []string
|
|
seen := map[string]bool{}
|
|
secret, provision := false, false
|
|
for _, w := range waits {
|
|
var act string
|
|
switch {
|
|
case w.Secret != "":
|
|
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
|
case w.Setting != "":
|
|
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
|
case w.Provision != "":
|
|
// ADR 0337 §7: "give <what> of <provider> on <machine>"; the line is in the summary.
|
|
act, provision = "give "+w.What, true
|
|
}
|
|
if act != "" && !seen[act] {
|
|
seen[act] = true
|
|
acts = append(acts, act)
|
|
}
|
|
}
|
|
needs := strings.Join(acts, "; and ") + "."
|
|
// Plain words hold one sentence of at most conditions.NeedsMax characters: several acts are named in the
|
|
// summary instead.
|
|
if len(acts) == 0 || len(needs) > conditions.NeedsMax {
|
|
needs = fmt.Sprintf("give what %s waits for on %s; the details name each secret and setting.", module, node)
|
|
}
|
|
explanation := fmt.Sprintf("Part of %s on %s cannot work until you give what it waits for.", module, node)
|
|
if secret {
|
|
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
|
"is sealed to the machine."
|
|
}
|
|
if provision {
|
|
explanation += " A key only you can give is given once for the app that issued it, and reaches every " +
|
|
"module that uses it."
|
|
}
|
|
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
|
return words{
|
|
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
|
Needs: needs,
|
|
Explanation: explanation,
|
|
Resolved: fmt.Sprintf("%s on %s no longer waits for you", module, node),
|
|
}
|
|
}
|
|
|
|
// readWaitFacts reads what the controller holds to check the waits of the modules named on one machine: the
|
|
// manifests (the catalogue's, or those given) and the secrets given there. A read that fails leaves that module
|
|
// unknown, so none of its waits is excused.
|
|
func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string, modules []string,
|
|
manifests map[string]catalogue.Manifest, f *operatorWaitFacts) {
|
|
if f.manifests == nil {
|
|
f.manifests = map[string]catalogue.Manifest{}
|
|
}
|
|
if f.given == nil {
|
|
f.given = map[string]map[string]time.Time{}
|
|
}
|
|
if f.outside == nil {
|
|
f.outside = map[string]outsideProvider{}
|
|
}
|
|
if f.pairsGiven == nil {
|
|
f.pairsGiven = map[string]map[string]bool{}
|
|
}
|
|
if inv == nil {
|
|
return
|
|
}
|
|
var shelf map[string]catalogue.Manifest
|
|
sort.Strings(modules)
|
|
for _, module := range modules {
|
|
if _, has := f.manifests[module]; !has {
|
|
if m, given := manifests[module]; given {
|
|
f.manifests[module] = m
|
|
} else {
|
|
if shelf == nil {
|
|
var err error
|
|
if shelf, err = inv.Catalogue(ctx); err != nil {
|
|
shelf = map[string]catalogue.Manifest{}
|
|
}
|
|
}
|
|
if m, known := shelf[module]; known {
|
|
f.manifests[module] = m
|
|
}
|
|
}
|
|
}
|
|
if _, read := f.given[module+"@"+machine]; !read {
|
|
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
|
f.given[module+"@"+machine] = given
|
|
}
|
|
}
|
|
if _, read := f.pairsGiven[module+"@"+machine]; !read {
|
|
if given, err := inv.GivenPairsOf(ctx, machine, module); err == nil {
|
|
f.pairsGiven[module+"@"+machine] = given
|
|
}
|
|
}
|
|
// The provisions it requires whose credential only a person gives, and where their provider runs (ADR 0337).
|
|
m, known := f.manifests[module]
|
|
if !known {
|
|
continue
|
|
}
|
|
for _, provision := range m.Requires {
|
|
if _, read := f.outside[provision]; read {
|
|
continue
|
|
}
|
|
if shelf == nil {
|
|
var err error
|
|
if shelf, err = inv.Catalogue(ctx); err != nil {
|
|
shelf = map[string]catalogue.Manifest{}
|
|
}
|
|
}
|
|
provider, outside := catalogue.OffersIssuedOutside(shelf, provision)
|
|
if !outside {
|
|
continue
|
|
}
|
|
f.outside[provision] = outsideProvider{Module: provider, Machines: machinesRunning(ctx, inv, provider)}
|
|
}
|
|
}
|
|
}
|
|
|
|
// machinesRunning is every machine a module is assigned to, by name; a read that fails is left out.
|
|
func machinesRunning(ctx context.Context, inv *inventory.Inventory, module string) []string {
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
// Only the summary's accept line reads it, which then names the provider's machine "<its machine>".
|
|
// empty-on-error: no wait is excused or refused by it
|
|
return nil
|
|
}
|
|
var out []string
|
|
for _, n := range nodes {
|
|
assigned, err := inv.Assigned(ctx, n.Name)
|
|
if err == nil && slices.Contains(assigned, module) {
|
|
out = append(out, n.Name)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// waitingModules is every module with a waiting resource in a statement.
|
|
func waitingModules(rs []inventory.ResourceHealth) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range rs {
|
|
if r.State == link.StateWaiting && r.Module != "" && !seen[r.Module] {
|
|
seen[r.Module] = true
|
|
out = append(out, r.Module)
|
|
}
|
|
}
|
|
return out
|
|
}
|