package assign import ( "fmt" "mesh-delivery-assigner/internal/assignment" "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" ) // The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node. const ( OptionRestore = "restore" OptionLeaveStopped = "leave-stopped" ) // The options of the outside ask (design 47 G §706). const ( OptionGoOn = "go-on" OptionFailed = "failed" ) // OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the // operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary // rule has it). about is the ask's About, `..`. func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask { return assignerAsk(asking, oc.Ask{ ID: meshdeliveryclient.AskingID(key, asking), Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause), Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+ "Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.", module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module), Options: []oc.Option{ {ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge}, {ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge}, }, Who: occ.Operator, Expires: occ.FormatTime(now.Add(AskLasts)), OnExpiry: "the delivery keeps waiting", About: about, }) } // WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery, // id `.`, severity warning, naming the module, the machine and the cause by its condition's key, and needing // nothing, so it is sent to no one: the operator's one message is the ask at the bound. func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed { return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node, Severity: "warning", Summary: fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey), Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node), Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+ "waits, judges again when it clears and goes on by itself.", module, node, causeKey), Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"} } // The options of the record ask (ADR 0539 §14) and of the backup ask (§16). const ( OptionYes = "yes" OptionNo = "no" OptionBackupAgain = mwc.BackupAgain OptionBackupSkip = mwc.BackupSkip ) // The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data. const ( OptionRevertHold = "hold" OptionRevertLive = mwc.RevertLive ) // KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `.`. const KindRevertHeld = "revert-held" // RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G // §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form // the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of // the asking, 0 the first, as the backup ask's. func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask { return assignerAsk(asking, oc.Ask{ ID: meshdeliveryclient.AskingID(key, asking), Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node), Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+ "%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.", module, node, short(fp), kept, form, words, module), Options: []oc.Option{ {ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge}, {ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge}, }, Who: occ.Operator, Expires: occ.FormatTime(now.Add(AskLasts)), OnExpiry: "the module keeps running its newer build", About: key, }) } // RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710). func RevertAnswerWords(option string, asking int) string { return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1) } // RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope // delivery, id `.`, a warning naming the reason, the kept data and its form, and needing nothing, so the // operator's one message is the ask. func RevertHeldFinding(node, module, kept, form, reason string) Observed { return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning", Summary: fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form), Headline: fmt.Sprintf("The revert of %s on %s is held", module, node), Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.", module, reason, kept, form, module), Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld} } // RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the // controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only // after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken // away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never // asked again, for the rewritten key carried the id the router already kept). func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask { what := map[Judgement]string{ JudgedRecreates: "recreates a container", JudgedMovesData: "moves the module's data", JudgedCannot: "cannot be judged: " + need.Words, }[need.Judgement] return assignerAsk(asking, oc.Ask{ ID: meshdeliveryclient.AskingID(key, asking), Headline: fmt.Sprintf("%s waits for your yes on %s", module, node), Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+ "The node is told to apply it only after you say yes.", module, node, short(fp), what), Options: []oc.Option{ {ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge}, {ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge}, }, Who: occ.Operator, Expires: occ.FormatTime(now.Add(AskLasts)), OnExpiry: "the node keeps what it runs", About: key, }) } // AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered // (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking. func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) } // NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1 // give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2. func NextBackupAsking(after int) int { if after < 0 { return 0 } return max(after+1, 2) } // BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want // value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries. func BackupAnswerWords(option string, asking int) string { return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1) } // BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on // without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the // backup's own words, from the node's have. func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask { return assignerAsk(asking, oc.Ask{ ID: meshdeliveryclient.AskingID(key, asking), Headline: fmt.Sprintf("The backup of %s failed on %s", module, node), Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.", module, short(fp), node, words, module), Options: []oc.Option{ {ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge}, {ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge}, }, Who: occ.Operator, Expires: occ.FormatTime(now.Add(AskLasts)), OnExpiry: "the module keeps running what it has", About: key, }) } // BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16), // naming the node, the module and the backup's words; cleared under the same key. func BackupCondition(node, module, fp, words string, raised bool) oc.Condition { key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2) subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node} if !raised { return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared} } headline := fmt.Sprintf("The backup of %s failed on %s", module, node) needs := "answer whether to back up again or go on without a backup" return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised, Summary: headline, Headline: headline, Needs: needs, Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."} }