package assign import ( "fmt" "regexp" "strings" "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" ) // HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or // release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held // record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a // larger number an ask written again after a hold lifted by itself, so the router takes it as new. func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk { return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{ ID: delivery.AskingID(key, asking), Headline: fmt.Sprintf("%s is held at %s", module, short(fp)), Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+ "no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+ "lets the wants move to it now.", module, node, short(fp), words, module), Options: []asks.Option{ {ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge}, }, Who: asks.Operator, Expires: now.Add(7 * 24 * time.Hour), OnExpiry: "the version stays held", About: key, }} } // The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node. const ( OptionRestore = "restore" OptionLeaveStopped = "leave-stopped" ) // RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is // not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the // node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is. func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk { return delivery.AssignerAsk{Ask: asks.Ask{ ID: delivery.AskingID(key, 0), Headline: fmt.Sprintf("%s failed its health check on %s", module, node), Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+ "would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words), Options: []asks.Option{ {ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge}, {ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge}, }, Who: asks.Operator, Expires: now.Add(7 * 24 * time.Hour), OnExpiry: "the module stays as it is on the node", About: key, }} } // HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or // clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's // words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding // data, whose restore ask waits for the operator. func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition { key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2) subject := asks.Scope{Scope: "module", ID: module, Machine: node} if !raised { return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared} } c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised, Summary: fmt.Sprintf("%s is held at %s", module, short(fp)), Headline: fmt.Sprintf("%s is held at %s", module, short(fp))} what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words) if needsYou { c.Needs = "answer whether to restore " + module + " on " + node c.Explanation = asks.NeedsYou + c.Needs + ". " + what } else { c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy." } return c } // NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition // gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent) // naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by // node and module, and cleared (raised false) when the node has a want for it again. func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition { key := delivery.AssignerModule + ".neither." + node + "." + module subject := asks.Scope{Scope: "module", ID: module, Machine: node} if !raised { return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)} } headline := fmt.Sprintf("%s on %s is on no list", module, node) if n := len([]rune(headline)); n > asks.HeadlineLength { headline = string([]rune(headline)[:asks.HeadlineLength]) } return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised, Summary: headline, Headline: headline, Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+ "hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+ "rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)} } // LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906). func LiftedCondition(module, fp string) asks.Condition { c := HeldCondition(module, fp, "", "", false, false) c.Resolved = LiftedWords(module, fp) + "." return c } // The options of the record ask (ADR 0539 §14) and of the backup ask (§16). const ( OptionYes = "yes" OptionNo = "no" OptionBackupAgain = delivery.BackupAgain OptionBackupSkip = delivery.BackupSkip ) // RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the // controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only // after the answer yes. The way the move was judged is in the ask. func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk { what := map[delivery.Judgement]string{ delivery.JudgedRecreates: "recreates a container", delivery.JudgedMovesData: "moves the module's data", delivery.JudgedCannot: "cannot be judged: " + need.Words, }[need.Judgement] return delivery.AssignerAsk{Ask: asks.Ask{ ID: delivery.AskingID(key, 0), Headline: fmt.Sprintf("%s waits for your yes on %s", module, node), Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+ "The node is told to apply it only after you say yes.", module, node, short(fp), what), Options: []asks.Option{ {ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge}, {ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge}, }, Who: asks.Operator, Expires: now.Add(7 * 24 * time.Hour), OnExpiry: "the node keeps what it runs", About: key, }} } // NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1 // give one id (delivery.AskingID), so the first asking is 0 and the next 2. func NextBackupAsking(after int) int { if after < 0 { return 0 } return max(after+1, 2) } var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`) // BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names // none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again // under a new number (ADR 0539 §16). func BackupAskDigest(a delivery.AssignerAsk) string { if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil { return m[1] } return "" } // BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want // value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries. func BackupAnswerWords(option string, asking int) string { return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1) } // BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on // without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the // backup's own words, from the node's have. func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk { return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{ ID: delivery.AskingID(key, asking), Headline: fmt.Sprintf("The backup of %s failed on %s", module, node), Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)", module, short(fp), node, words, module, digest), Options: []asks.Option{ {ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge}, {ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge}, }, Who: asks.Operator, Expires: now.Add(7 * 24 * time.Hour), OnExpiry: "the module keeps running what it has", About: key, }} } // BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16), // naming the node, the module and the backup's words; cleared under the same key. func BackupCondition(node, module, fp, words string, raised bool) asks.Condition { key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2) subject := asks.Scope{Scope: "module", ID: module, Machine: node} if !raised { return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared} } headline := fmt.Sprintf("The backup of %s failed on %s", module, node) needs := "answer whether to back up again or go on without a backup" return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised, Summary: headline, Headline: headline, Needs: needs, Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."} }