mesh-delivery/modules/mesh-delivery-assigner/internal/assign/asks.go

package assign

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/asks"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
	return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
		ID:       delivery.AskingID(key, asking),
		Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
		Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
			"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
			"lets the wants move to it now.", module, node, short(fp), words, module),
		Options: []asks.Option{
			{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
			{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
		},
		Who:      asks.Operator,
		Expires:  now.Add(7 * 24 * time.Hour),
		OnExpiry: "the version stays held",
		About:    key,
	}}
}

// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
const (
	OptionRestore      = "restore"
	OptionLeaveStopped = "leave-stopped"
)

// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
	return delivery.AssignerAsk{Ask: asks.Ask{
		ID:       delivery.AskingID(key, 0),
		Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
		Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
			"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
		Options: []asks.Option{
			{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
			{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
		},
		Who:      asks.Operator,
		Expires:  now.Add(7 * 24 * time.Hour),
		OnExpiry: "the module stays as it is on the node",
		About:    key,
	}}
}

// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
	key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
	subject := asks.Scope{Scope: "module", ID: module, Machine: node}
	if !raised {
		return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
	}
	c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
		Summary:  fmt.Sprintf("%s is held at %s", module, short(fp)),
		Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
	what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
	if needsYou {
		c.Needs = "answer whether to restore " + module + " on " + node
		c.Explanation = asks.NeedsYou + c.Needs + ". " + what
	} else {
		c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
	}
	return c
}

// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
	key := delivery.AssignerModule + ".failed-alone." + node + "." + module
	subject := asks.Scope{Scope: "module", ID: module, Machine: node}
	if !raised {
		return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
			Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
	}
	headline := fmt.Sprintf("%s fails on %s alone", module, node)
	if n := len([]rune(headline)); n > asks.HeadlineLength {
		headline = string([]rune(headline)[:asks.HeadlineLength])
	}
	return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
		Summary: headline, Headline: headline,
		Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
			"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
			"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
			node, node)}
}

// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
	key := delivery.AssignerModule + ".neither." + node + "." + module
	subject := asks.Scope{Scope: "module", ID: module, Machine: node}
	if !raised {
		return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
			Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
	}
	headline := fmt.Sprintf("%s on %s is on no list", module, node)
	if n := len([]rune(headline)); n > asks.HeadlineLength {
		headline = string([]rune(headline)[:asks.HeadlineLength])
	}
	if h.Bus {
		needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
		return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
			Summary: headline, Headline: headline, Needs: needs,
			Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
				"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
				asks.NeedsYou, needs, module, node, h.Words)}
	}
	return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
		Summary: headline, Headline: headline,
		Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
			"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
			"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}

// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) asks.Condition {
	c := NeitherCondition(node, module, Holding{}, false)
	if why != "" {
		c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
	}
	return c
}

// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
	c := HeldCondition(module, fp, "", "", false, false)
	c.Resolved = LiftedWords(module, fp) + "."
	return c
}

// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
	OptionYes         = "yes"
	OptionNo          = "no"
	OptionBackupAgain = delivery.BackupAgain
	OptionBackupSkip  = delivery.BackupSkip
)

// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
	what := map[delivery.Judgement]string{
		delivery.JudgedRecreates: "recreates a container",
		delivery.JudgedMovesData: "moves the module's data",
		delivery.JudgedCannot:    "cannot be judged: " + need.Words,
	}[need.Judgement]
	return delivery.AssignerAsk{Ask: asks.Ask{
		ID:       delivery.AskingID(key, 0),
		Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
		Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
			"The node is told to apply it only after you say yes.", module, node, short(fp), what),
		Options: []asks.Option{
			{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
			{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
		},
		Who:      asks.Operator,
		Expires:  now.Add(7 * 24 * time.Hour),
		OnExpiry: "the node keeps what it runs",
		About:    key,
	}}
}

// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
	if after < 0 {
		return 0
	}
	return max(after+1, 2)
}

var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)

// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a delivery.AssignerAsk) string {
	if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
		return m[1]
	}
	return ""
}

// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
func BackupAnswerWords(option string, asking int) string {
	return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
}

// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
	return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
		ID:       delivery.AskingID(key, asking),
		Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
		Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
			module, short(fp), node, words, module, digest),
		Options: []asks.Option{
			{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
			{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
		},
		Who:      asks.Operator,
		Expires:  now.Add(7 * 24 * time.Hour),
		OnExpiry: "the module keeps running what it has",
		About:    key,
	}}
}

// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
	key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
	subject := asks.Scope{Scope: "module", ID: module, Machine: node}
	if !raised {
		return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
	}
	headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
	needs := "answer whether to back up again or go on without a backup"
	return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
		Summary: headline, Headline: headline, Needs: needs,
		Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}