With no callers list on the seat mesh-wants, the asker a step's progress names is the one the runner writes on the StepRun it asks (C-W2), mesh-pipeline-runner. Builds: design 47 H §752 (consequence), G §496; H-A-41, H-A-88 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
200 lines
11 KiB
Go
200 lines
11 KiB
Go
package assign
|
|
|
|
import (
|
|
"fmt"
|
|
"mesh-delivery-assigner/internal/assignment"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
|
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
|
|
|
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
|
)
|
|
|
|
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
|
const (
|
|
OptionRestore = "restore"
|
|
OptionLeaveStopped = "leave-stopped"
|
|
)
|
|
|
|
// The options of the outside ask (design 47 G §706).
|
|
const (
|
|
OptionGoOn = "go-on"
|
|
OptionFailed = "failed"
|
|
)
|
|
|
|
// OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the
|
|
// operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary
|
|
// rule has it). about is the ask's About, `<node>.<module>.<fingerprint hex>`.
|
|
func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask {
|
|
return assignerAsk(asking, oc.Ask{
|
|
ID: meshdeliveryclient.AskingID(key, asking),
|
|
Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause),
|
|
Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+
|
|
"Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.",
|
|
module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module),
|
|
Options: []oc.Option{
|
|
{ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge},
|
|
{ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge},
|
|
},
|
|
Who: occ.Operator,
|
|
Expires: occ.FormatTime(now.Add(AskLasts)),
|
|
OnExpiry: "the delivery keeps waiting",
|
|
About: about,
|
|
})
|
|
}
|
|
|
|
// WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery,
|
|
// id `<node>.<module>`, severity warning, naming the module, the machine and the cause by its condition's key, and needing
|
|
// nothing, so it is sent to no one: the operator's one message is the ask at the bound.
|
|
func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed {
|
|
return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node,
|
|
Severity: "warning",
|
|
Summary: fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey),
|
|
Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node),
|
|
Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+
|
|
"waits, judges again when it clears and goes on by itself.", module, node, causeKey),
|
|
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
|
|
}
|
|
|
|
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
|
const (
|
|
OptionYes = "yes"
|
|
OptionNo = "no"
|
|
OptionBackupAgain = mwc.BackupAgain
|
|
OptionBackupSkip = mwc.BackupSkip
|
|
)
|
|
|
|
// The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data.
|
|
const (
|
|
OptionRevertHold = "hold"
|
|
OptionRevertLive = mwc.RevertLive
|
|
)
|
|
|
|
// KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `<node>.<module>`.
|
|
const KindRevertHeld = "revert-held"
|
|
|
|
// RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G
|
|
// §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form
|
|
// the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of
|
|
// the asking, 0 the first, as the backup ask's.
|
|
func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask {
|
|
return assignerAsk(asking, oc.Ask{
|
|
ID: meshdeliveryclient.AskingID(key, asking),
|
|
Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node),
|
|
Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+
|
|
"%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.",
|
|
module, node, short(fp), kept, form, words, module),
|
|
Options: []oc.Option{
|
|
{ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge},
|
|
{ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge},
|
|
},
|
|
Who: occ.Operator,
|
|
Expires: occ.FormatTime(now.Add(AskLasts)),
|
|
OnExpiry: "the module keeps running its newer build",
|
|
About: key,
|
|
})
|
|
}
|
|
|
|
// RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710).
|
|
func RevertAnswerWords(option string, asking int) string {
|
|
return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1)
|
|
}
|
|
|
|
// RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope
|
|
// delivery, id `<node>.<module>`, a warning naming the reason, the kept data and its form, and needing nothing, so the
|
|
// operator's one message is the ask.
|
|
func RevertHeldFinding(node, module, kept, form, reason string) Observed {
|
|
return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning",
|
|
Summary: fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form),
|
|
Headline: fmt.Sprintf("The revert of %s on %s is held", module, node),
|
|
Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.",
|
|
module, reason, kept, form, module),
|
|
Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld}
|
|
}
|
|
|
|
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
|
|
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
|
|
// after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken
|
|
// away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never
|
|
// asked again, for the rewritten key carried the id the router already kept).
|
|
func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask {
|
|
what := map[Judgement]string{
|
|
JudgedRecreates: "recreates a container",
|
|
JudgedMovesData: "moves the module's data",
|
|
JudgedCannot: "cannot be judged: " + need.Words,
|
|
}[need.Judgement]
|
|
return assignerAsk(asking, oc.Ask{
|
|
ID: meshdeliveryclient.AskingID(key, asking),
|
|
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
|
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
|
Options: []oc.Option{
|
|
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
|
|
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
|
|
},
|
|
Who: occ.Operator,
|
|
Expires: occ.FormatTime(now.Add(AskLasts)),
|
|
OnExpiry: "the node keeps what it runs",
|
|
About: key,
|
|
})
|
|
}
|
|
|
|
// AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered
|
|
// (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking.
|
|
func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) }
|
|
|
|
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
|
|
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
|
|
func NextBackupAsking(after int) int {
|
|
if after < 0 {
|
|
return 0
|
|
}
|
|
return max(after+1, 2)
|
|
}
|
|
|
|
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
|
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
|
func BackupAnswerWords(option string, asking int) string {
|
|
return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
|
|
}
|
|
|
|
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
|
// without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the
|
|
// backup's own words, from the node's have.
|
|
func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
|
|
return assignerAsk(asking, oc.Ask{
|
|
ID: meshdeliveryclient.AskingID(key, asking),
|
|
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
|
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.",
|
|
module, short(fp), node, words, module),
|
|
Options: []oc.Option{
|
|
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
|
|
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
|
|
},
|
|
Who: occ.Operator,
|
|
Expires: occ.FormatTime(now.Add(AskLasts)),
|
|
OnExpiry: "the module keeps running what it has",
|
|
About: key,
|
|
})
|
|
}
|
|
|
|
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
|
// naming the node, the module and the backup's words; cleared under the same key.
|
|
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
|
|
key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
|
|
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
|
if !raised {
|
|
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
|
|
}
|
|
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
|
needs := "answer whether to back up again or go on without a backup"
|
|
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
|
Summary: headline, Headline: headline, Needs: needs,
|
|
Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
|
}
|