147 lines
8.0 KiB
Go
147 lines
8.0 KiB
Go
package assign
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
|
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
|
|
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
|
ID: delivery.AskingID(key, 0),
|
|
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
|
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
|
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
|
|
Options: []asks.Option{
|
|
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
|
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
|
},
|
|
Who: asks.Operator,
|
|
Expires: now.Add(7 * 24 * time.Hour),
|
|
OnExpiry: "the version stays held",
|
|
About: key,
|
|
}}
|
|
}
|
|
|
|
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
|
const (
|
|
OptionRestore = "restore"
|
|
OptionLeaveStopped = "leave-stopped"
|
|
)
|
|
|
|
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
|
|
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
|
|
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
|
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
|
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
|
ID: delivery.AskingID(key, 0),
|
|
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
|
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
|
Options: []asks.Option{
|
|
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
|
|
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
|
|
},
|
|
Who: asks.Operator,
|
|
Expires: now.Add(7 * 24 * time.Hour),
|
|
OnExpiry: "the module stays as it is on the node",
|
|
About: key,
|
|
}}
|
|
}
|
|
|
|
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
|
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
|
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
|
// data, whose restore ask waits for the operator.
|
|
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
|
|
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
|
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
|
if !raised {
|
|
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
|
}
|
|
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
|
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
|
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
|
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
|
if needsYou {
|
|
c.Needs = "answer whether to restore " + module + " on " + node
|
|
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
|
|
} else {
|
|
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
|
}
|
|
return c
|
|
}
|
|
|
|
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
|
const (
|
|
OptionYes = "yes"
|
|
OptionNo = "no"
|
|
OptionBackupAgain = delivery.BackupAgain
|
|
OptionBackupSkip = delivery.BackupSkip
|
|
)
|
|
|
|
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
|
|
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
|
|
// after the answer yes. The way the move was judged is in the ask.
|
|
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
|
|
what := map[delivery.Judgement]string{
|
|
delivery.JudgedRecreates: "recreates a container",
|
|
delivery.JudgedMovesData: "moves the module's data",
|
|
delivery.JudgedCannot: "cannot be judged: " + need.Words,
|
|
}[need.Judgement]
|
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
|
ID: delivery.AskingID(key, 0),
|
|
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
|
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
|
Options: []asks.Option{
|
|
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
|
|
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
|
|
},
|
|
Who: asks.Operator,
|
|
Expires: now.Add(7 * 24 * time.Hour),
|
|
OnExpiry: "the node keeps what it runs",
|
|
About: key,
|
|
}}
|
|
}
|
|
|
|
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
|
// without a backup. asking is the number of the asking (0 the first, 2 the second: delivery.AskingID), so a backup that fails again after the answer again is asked
|
|
// again. words are the backup's own words, from the node's have.
|
|
func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) delivery.AssignerAsk {
|
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
|
ID: delivery.AskingID(key, asking),
|
|
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
|
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.",
|
|
module, short(fp), node, words, module),
|
|
Options: []asks.Option{
|
|
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
|
|
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
|
|
},
|
|
Who: asks.Operator,
|
|
Expires: now.Add(7 * 24 * time.Hour),
|
|
OnExpiry: "the module keeps running what it has",
|
|
About: key,
|
|
}}
|
|
}
|
|
|
|
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
|
// naming the node, the module and the backup's words; cleared under the same key.
|
|
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
|
|
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
|
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
|
if !raised {
|
|
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
|
}
|
|
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
|
needs := "answer whether to back up again or go on without a backup"
|
|
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
|
Summary: headline, Headline: headline, Needs: needs,
|
|
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
|
}
|