mesh-delivery/modules/mesh-delivery-assigner/internal/assign/asks.go

package assign

import (
	"fmt"
	"mesh-delivery-assigner/internal/assignment"
	"regexp"
	"strings"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
	mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"

	occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
	oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)

// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask {
	return assignerAsk(asking, oc.Ask{
		ID:       meshdeliveryclient.AskingID(key, asking),
		Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
		Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
			"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
			"lets the wants move to it now.", module, node, short(fp), words, module),
		Options: []oc.Option{
			{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge},
			{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge},
		},
		Who:      occ.Operator,
		Expires:  occ.FormatTime(now.Add(AskLasts)),
		OnExpiry: "the version stays held",
		About:    key,
	})
}

// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
const (
	OptionRestore      = "restore"
	OptionLeaveStopped = "leave-stopped"
)

// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. asking is the writing's
// number (AskingSeconds): an ask taken away as stale and written again is another asking under an id of its own, since the
// router drops a second ask under an id it keeps and the teller took the first back (issue 1966). Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
	return assignerAsk(asking, oc.Ask{
		ID:       meshdeliveryclient.AskingID(key, asking),
		Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
		Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
			"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
		Options: []oc.Option{
			{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge},
			{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge},
		},
		Who:      occ.Operator,
		Expires:  occ.FormatTime(now.Add(AskLasts)),
		OnExpiry: "the module stays as it is on the node",
		About:    key,
	})
}

// The options of the outside ask (design 47 G §706).
const (
	OptionGoOn   = "go-on"
	OptionFailed = "failed"
)

// OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the
// operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary
// rule has it). about is the ask's About, `<node>.<module>.<fingerprint hex>`.
func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask {
	return assignerAsk(asking, oc.Ask{
		ID:       meshdeliveryclient.AskingID(key, asking),
		Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause),
		Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+
			"Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.",
			module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module),
		Options: []oc.Option{
			{ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge},
			{ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge},
		},
		Who:      occ.Operator,
		Expires:  occ.FormatTime(now.Add(AskLasts)),
		OnExpiry: "the delivery keeps waiting",
		About:    about,
	})
}

// WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery,
// id `<node>.<module>`, severity warning, naming the module, the machine and the cause by its condition's key, and needing
// nothing, so it is sent to no one: the operator's one message is the ask at the bound.
func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed {
	return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node,
		Severity: "warning",
		Summary:  fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey),
		Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node),
		Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+
			"waits, judges again when it clears and goes on by itself.", module, node, causeKey),
		Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
}

// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
	key := assignment.Module() + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
	subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
	if !raised {
		return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
	}
	c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
		Summary:  fmt.Sprintf("%s is held at %s", module, short(fp)),
		Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
	what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
	if needsYou {
		c.Needs = "answer whether to restore " + module + " on " + node
		c.Explanation = occ.NeedsYou + c.Needs + ". " + what
	} else {
		c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
	}
	return c
}

// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition {
	key := assignment.Module() + ".failed-alone." + node + "." + module
	subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
	if !raised {
		return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
			Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
	}
	headline := fmt.Sprintf("%s fails on %s alone", module, node)
	if n := len([]rune(headline)); n > occ.HeadlineLength {
		headline = string([]rune(headline)[:occ.HeadlineLength])
	}
	return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
		Summary: headline, Headline: headline,
		Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
			"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
			"another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
			node, node)}
}

// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition {
	key := assignment.Module() + ".neither." + node + "." + module
	subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
	if !raised {
		return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
			Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
	}
	headline := fmt.Sprintf("%s on %s is on no list", module, node)
	if n := len([]rune(headline)); n > occ.HeadlineLength {
		headline = string([]rune(headline)[:occ.HeadlineLength])
	}
	return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised,
		Summary: headline, Headline: headline,
		Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
			"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
			"rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)}
}

// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) oc.Condition {
	c := NeitherCondition(node, module, Holding{}, false)
	if why != "" {
		c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
	}
	return c
}

// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) oc.Condition {
	c := HeldCondition(module, fp, "", "", false, false)
	c.Resolved = LiftedWords(module, fp) + "."
	return c
}

// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
	OptionYes         = "yes"
	OptionNo          = "no"
	OptionBackupAgain = mwc.BackupAgain
	OptionBackupSkip  = mwc.BackupSkip
)

// The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data.
const (
	OptionRevertHold = "hold"
	OptionRevertLive = mwc.RevertLive
)

// KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `<node>.<module>`.
const KindRevertHeld = "revert-held"

// RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G
// §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form
// the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of
// the asking, 0 the first, as the backup ask's.
func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask {
	return assignerAsk(asking, oc.Ask{
		ID:       meshdeliveryclient.AskingID(key, asking),
		Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node),
		Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+
			"%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.",
			module, node, short(fp), kept, form, words, module),
		Options: []oc.Option{
			{ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge},
			{ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge},
		},
		Who:      occ.Operator,
		Expires:  occ.FormatTime(now.Add(AskLasts)),
		OnExpiry: "the module keeps running its newer build",
		About:    key,
	})
}

// RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710).
func RevertAnswerWords(option string, asking int) string {
	return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1)
}

// RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope
// delivery, id `<node>.<module>`, a warning naming the reason, the kept data and its form, and needing nothing, so the
// operator's one message is the ask.
func RevertHeldFinding(node, module, kept, form, reason string) Observed {
	return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning",
		Summary:  fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form),
		Headline: fmt.Sprintf("The revert of %s on %s is held", module, node),
		Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.",
			module, reason, kept, form, module),
		Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld}
}

// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken
// away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never
// asked again, for the rewritten key carried the id the router already kept).
func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask {
	what := map[Judgement]string{
		JudgedRecreates: "recreates a container",
		JudgedMovesData: "moves the module's data",
		JudgedCannot:    "cannot be judged: " + need.Words,
	}[need.Judgement]
	return assignerAsk(asking, oc.Ask{
		ID:       meshdeliveryclient.AskingID(key, asking),
		Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
		Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
			"The node is told to apply it only after you say yes.", module, node, short(fp), what),
		Options: []oc.Option{
			{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
			{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
		},
		Who:      occ.Operator,
		Expires:  occ.FormatTime(now.Add(AskLasts)),
		OnExpiry: "the node keeps what it runs",
		About:    key,
	})
}

// AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered
// (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking.
func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) }

// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
	if after < 0 {
		return 0
	}
	return max(after+1, 2)
}

var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)

// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a asks.Ask) string {
	ask, err := AskOf(a)
	if err != nil {
		return ""
	}
	if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil {
		return m[1]
	}
	return ""
}

// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
func BackupAnswerWords(option string, asking int) string {
	return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
}

// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask {
	return assignerAsk(asking, oc.Ask{
		ID:       meshdeliveryclient.AskingID(key, asking),
		Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
		Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
			module, short(fp), node, words, module, digest),
		Options: []oc.Option{
			{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
			{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
		},
		Who:      occ.Operator,
		Expires:  occ.FormatTime(now.Add(AskLasts)),
		OnExpiry: "the module keeps running what it has",
		About:    key,
	})
}

// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
	key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
	subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
	if !raised {
		return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
	}
	headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
	needs := "answer whether to back up again or go on without a backup"
	return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
		Summary: headline, Headline: headline, Needs: needs,
		Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}