jschoubben
d6cef5829b
The node agent applies only the composition its want names, and the assigner writes a want only once that composition exists (ADR 0539 clauses 1, 6, 14 and 16).
501 lines
20 KiB
Go
501 lines
20 KiB
Go
package delivery
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
)
|
|
|
|
// The assigner's keys (novox/hq ADR 0528 clauses 1 and 4, ADR 0429 §6, issue 1297). The assigner writes, by the six
|
|
// rules of ADR 0429 §6, what each node should run: `want.<node>.<module>` in its bucket wants, and beside it
|
|
// `held.<module>.<fingerprint>` for a fingerprint it holds after a failed health check. A module that leaves a node's
|
|
// modules gets its want written removed, never deleted (ADR 0408 §4). For each node and module it compares the want with
|
|
// what the controller sends (the build the controller's assignment names) and writes every difference under
|
|
// `<node>.<module>` in its bucket differences, with the rule that explains it or unexplained. In release 5 wants are
|
|
// only compared; the node agent applying them is release 6 (ADR 0429 §7). Only the assigner's credential writes these
|
|
// buckets; the watchman, the view and the controller read them.
|
|
//
|
|
// **Release 6** (ADR 0539 clauses 1, 6 and 16): a want also names the digest of the one composition it allows, written
|
|
// only once that composition exists (the seventh rule, RuleComposed), and carries the answer to a failed backup; the
|
|
// node's own want `want.<node>.node` names the digest of the node's own composition (NodeWant).
|
|
//
|
|
// **The bucket is the first word, the key the rest** (as the builder's `builds.<module>.<fp>` is the key
|
|
// `<module>.<fp>` in its builds bucket): the differences bucket's keys are `<node>.<module>`. The wants bucket holds
|
|
// two kinds of key, so each keeps its word: `want.<node>.<module>` and `held.<module>.<fingerprint hex>`.
|
|
const (
|
|
// AssignerModule is the module of the assigner part.
|
|
AssignerModule = "mesh-delivery-assigner"
|
|
// AssignerWantsBucket is the assigner's bucket of wants and holds, `<owner module>_<name>`.
|
|
AssignerWantsBucket = AssignerModule + "_wants"
|
|
// AssignerWantsHistory is the revisions the wants bucket keeps per key, so a want's history shows each move and
|
|
// the operator's manual write (ADR 0429 §6, the one escape).
|
|
AssignerWantsHistory = 20
|
|
// AssignerDifferencesBucket is the assigner's bucket of differences between a want and what the controller sends.
|
|
AssignerDifferencesBucket = AssignerModule + "_differences"
|
|
// AssignerDifferencesHistory is the revisions the differences bucket keeps per key: acceptance test 17 of design 47
|
|
// reads a day of them.
|
|
AssignerDifferencesHistory = 64
|
|
)
|
|
|
|
// The words that open the wants bucket's two kinds of key.
|
|
const (
|
|
wantWord = "want"
|
|
heldWord = "held"
|
|
)
|
|
|
|
// WantKey is the key, in AssignerWantsBucket, of a node's want for a module: `want.<node>.<module>`. A module named
|
|
// `node` is refused, since `want.<node>.node` is the node's own key (ADR 0429 §4), as HaveKey refuses it.
|
|
func WantKey(node, module string) (string, error) {
|
|
have, err := HaveKey(node, module)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return wantWord + "." + have, nil
|
|
}
|
|
|
|
// WantNodeKey is the key, in AssignerWantsBucket, of a node's own want: `want.<node>.node`, naming the digest of the
|
|
// node's own composition (ADR 0539 clause 6; ADR 0429 §4's node key). The assigner writes it once the node's own
|
|
// composition exists.
|
|
func WantNodeKey(node string) (string, error) {
|
|
own, err := HaveNodeKey(node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return wantWord + "." + own, nil
|
|
}
|
|
|
|
// ParseWantNodeKey is the inverse of WantNodeKey; it refuses a module's want key and any key of another bucket,
|
|
// naming it.
|
|
func ParseWantNodeKey(key string) (node string, err error) {
|
|
rest, ok := strings.CutPrefix(key, wantWord+".")
|
|
if !ok {
|
|
return "", fmt.Errorf("delivery: %q is not a node's own want key, want.<node>.node", key)
|
|
}
|
|
node, _, isNode, err := ParseHaveKey(rest)
|
|
if err != nil || !isNode {
|
|
return "", fmt.Errorf("delivery: %q is not a node's own want key, want.<node>.node", key)
|
|
}
|
|
return node, nil
|
|
}
|
|
|
|
// ParseWantKey is the inverse of WantKey; it refuses a held key, a have key, the node's own want key and any key of
|
|
// another bucket, naming it.
|
|
func ParseWantKey(key string) (node, module string, err error) {
|
|
rest, ok := strings.CutPrefix(key, wantWord+".")
|
|
if !ok {
|
|
return "", "", fmt.Errorf("delivery: %q is not a want key, want.<node>.<module>", key)
|
|
}
|
|
node, module, isNode, err := ParseHaveKey(rest)
|
|
if err != nil || isNode {
|
|
return "", "", fmt.Errorf("delivery: %q is not a want key, want.<node>.<module>", key)
|
|
}
|
|
return node, module, nil
|
|
}
|
|
|
|
// HeldKey is the key, in AssignerWantsBucket, of a module's fingerprint the assigner holds: `held.<module>.<hex>`,
|
|
// the hex being the fingerprint's key part (FingerprintKeyPart).
|
|
func HeldKey(module, fingerprint string) (string, error) {
|
|
key, err := BuilderBuildKey(module, fingerprint)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return heldWord + "." + key, nil
|
|
}
|
|
|
|
// ParseHeldKey is the inverse of HeldKey: the module and the fingerprint (`src2:` and the hex); it refuses a want key
|
|
// and any key of another bucket, naming it.
|
|
func ParseHeldKey(key string) (module, fingerprint string, err error) {
|
|
rest, ok := strings.CutPrefix(key, heldWord+".")
|
|
if !ok {
|
|
return "", "", fmt.Errorf("delivery: %q is not a held key, held.<module>.<fingerprint hex>", key)
|
|
}
|
|
if module, fingerprint, err = ParseBuilderBuildKey(rest); err != nil {
|
|
return "", "", fmt.Errorf("delivery: %q is not a held key: %w", key, err)
|
|
}
|
|
return module, fingerprint, nil
|
|
}
|
|
|
|
// DifferenceKey is the key, in AssignerDifferencesBucket, of the difference for a module on a node: `<node>.<module>`.
|
|
func DifferenceKey(node, module string) (string, error) {
|
|
return HaveKey(node, module)
|
|
}
|
|
|
|
// ParseDifferenceKey is the inverse of DifferenceKey; it refuses a want or held key and a node's own key, naming it.
|
|
func ParseDifferenceKey(key string) (node, module string, err error) {
|
|
node, module, isNode, err := ParseHaveKey(key)
|
|
if err != nil || isNode {
|
|
return "", "", fmt.Errorf("delivery: %q is not a difference key, <node>.<module>", key)
|
|
}
|
|
return node, module, nil
|
|
}
|
|
|
|
// The rules of ADR 0429 §6 and ADR 0539 clause 6, by number, as a want or a difference names the one that holds a node
|
|
// back.
|
|
const (
|
|
RuleBuildDone = 1 // the build of F is done
|
|
RuleNotHeld = 2 // F is not held
|
|
RuleCanary = 3 // the node is the module's canary, or the canary runs F healthy
|
|
RuleStep = 4 // at most step other nodes are ahead (step 0: no limit, ADR 0528 §3)
|
|
RuleProviders = 5 // every provider the module requires runs healthy on that node, or at its canary
|
|
RuleGrant = 6 // the module's credential for F exists (grants.<module>.<fp>)
|
|
RuleComposed = 7 // the module's composition for that node at F exists (mesh-controller_composed, ADR 0539 §6)
|
|
)
|
|
|
|
func checkRule(rule int, of string) error {
|
|
if rule < RuleBuildDone || rule > RuleComposed {
|
|
return fmt.Errorf("delivery: %s names the rule %d, which is not one of the seven (1 to 7)", of, rule)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// The answers a want carries to a failed backup before an apply (ADR 0539 clause 16): the node agent reads them from
|
|
// its own want and from nothing else.
|
|
const (
|
|
// BackupAgain is the answer to back the module up again before the apply.
|
|
BackupAgain = "again"
|
|
// BackupSkip is the answer to go on without a backup.
|
|
BackupSkip = "skip"
|
|
)
|
|
|
|
func checkFingerprint(fp, of string) error {
|
|
if _, err := FingerprintKeyPart(fp); err != nil {
|
|
return fmt.Errorf("delivery: %s: %w", of, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Want is a node's want for a module: the fingerprint FP the node should run and the digest Composed of the one
|
|
// composition of it the node may apply (ADR 0539 clause 1), or Removed for a module that left the node's modules (then
|
|
// both are empty). When the module's newest fingerprint is not yet allowed on this node, Newest names it and Rule the
|
|
// rule that holds the node back; both are empty when FP is the newest. Backup is the answer to a failed backup before
|
|
// the apply, BackupAgain or BackupSkip (clause 16), empty when none was asked. Words say why in a sentence. The
|
|
// entry's own time says since when.
|
|
type Want struct {
|
|
V int `json:"v"`
|
|
FP string `json:"fp,omitempty"`
|
|
Composed string `json:"composed,omitempty"`
|
|
Removed bool `json:"removed,omitempty"`
|
|
Newest string `json:"newest,omitempty"`
|
|
Rule int `json:"rule,omitempty"`
|
|
Backup string `json:"backup,omitempty"`
|
|
Words string `json:"words,omitempty"`
|
|
}
|
|
|
|
func (w Want) check() error {
|
|
if w.Removed {
|
|
if w.FP != "" || w.Composed != "" || w.Newest != "" || w.Rule != 0 || w.Backup != "" {
|
|
return fmt.Errorf("delivery: a removed want names no fingerprint, composition, rule or backup answer")
|
|
}
|
|
return nil
|
|
}
|
|
if err := checkFingerprint(w.FP, "a want's fingerprint"); err != nil {
|
|
return err
|
|
}
|
|
if w.Composed == "" {
|
|
return fmt.Errorf("delivery: the want of %s names no composition: a want names the digest of the one composition it allows (ADR 0539 clause 1)", w.FP)
|
|
}
|
|
if err := checkDigest(w.Composed, "a want's composition digest"); err != nil {
|
|
return err
|
|
}
|
|
switch w.Backup {
|
|
case "", BackupAgain, BackupSkip:
|
|
default:
|
|
return fmt.Errorf("delivery: the backup answer %q is not %s or %s", w.Backup, BackupAgain, BackupSkip)
|
|
}
|
|
if w.Newest == "" && w.Rule == 0 {
|
|
return nil
|
|
}
|
|
if err := checkFingerprint(w.Newest, "a want's newest fingerprint"); err != nil {
|
|
return err
|
|
}
|
|
if w.Newest == w.FP {
|
|
return fmt.Errorf("delivery: a want at its newest fingerprint names no rule holding it back")
|
|
}
|
|
return checkRule(w.Rule, "a want held back")
|
|
}
|
|
|
|
// Held is a fingerprint the assigner holds: the node whose health check failed on it, and the check's words.
|
|
type Held struct {
|
|
V int `json:"v"`
|
|
Node string `json:"node"`
|
|
Words string `json:"words,omitempty"`
|
|
}
|
|
|
|
func (h Held) check() error {
|
|
return checkName("node", h.Node)
|
|
}
|
|
|
|
// Difference is where a want and what the controller sends differ for a module on a node (ADR 0528 clause 4): the
|
|
// want's fingerprint (Want, empty when the node has no want for the module, WantRemoved when the want is removed), the
|
|
// fingerprint of the build the controller's assignment names (Sent, empty when it sends none or the build names no
|
|
// fingerprint; SentBuild names that build), and the rule that explains the difference, or Unexplained.
|
|
type Difference struct {
|
|
V int `json:"v"`
|
|
Want string `json:"want,omitempty"`
|
|
WantRemoved bool `json:"want-removed,omitempty"`
|
|
Sent string `json:"sent,omitempty"`
|
|
SentBuild string `json:"sent-build,omitempty"`
|
|
Rule int `json:"rule,omitempty"`
|
|
Unexplained bool `json:"unexplained,omitempty"`
|
|
Words string `json:"words,omitempty"`
|
|
}
|
|
|
|
func (d Difference) check() error {
|
|
if d.WantRemoved && d.Want != "" {
|
|
return fmt.Errorf("delivery: a difference with a removed want names no wanted fingerprint")
|
|
}
|
|
if d.Want != "" {
|
|
if err := checkFingerprint(d.Want, "a difference's want"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if d.Sent != "" {
|
|
if err := checkFingerprint(d.Sent, "a difference's sent fingerprint"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !d.WantRemoved && d.Want == d.Sent && (d.Want != "" || d.SentBuild == "") {
|
|
return fmt.Errorf("delivery: a difference whose want and send are both %q is no difference", d.Want)
|
|
}
|
|
if d.Unexplained {
|
|
if d.Rule != 0 {
|
|
return fmt.Errorf("delivery: an unexplained difference names no rule")
|
|
}
|
|
return nil
|
|
}
|
|
return checkRule(d.Rule, "a difference")
|
|
}
|
|
|
|
// EncodeWant is w as JSON at Version; it refuses a fingerprint not of the new form, a removed want naming one, and a
|
|
// want held back without its rule.
|
|
func EncodeWant(w Want) ([]byte, error) {
|
|
if err := w.check(); err != nil {
|
|
return nil, err
|
|
}
|
|
w.V = Version
|
|
return json.Marshal(w)
|
|
}
|
|
|
|
// DecodeWant reads a want; it refuses an unknown version (ErrUnknownVersion) and what EncodeWant refuses.
|
|
func DecodeWant(data []byte) (Want, error) {
|
|
var w Want
|
|
if err := decode(data, &w); err != nil {
|
|
return Want{}, err
|
|
}
|
|
if err := w.check(); err != nil {
|
|
return Want{}, err
|
|
}
|
|
return w, nil
|
|
}
|
|
|
|
// SameWant says whether two wants are the same value, so the assigner writes only when they differ.
|
|
func SameWant(a, b Want) bool { return a == b }
|
|
|
|
// NodeWant is a node's own want (WantNodeKey): the digest Composed of the node's own composition it may apply. Words
|
|
// say why in a sentence.
|
|
type NodeWant struct {
|
|
V int `json:"v"`
|
|
Composed string `json:"composed"`
|
|
Words string `json:"words,omitempty"`
|
|
}
|
|
|
|
// EncodeNodeWant is w as JSON at Version; it refuses a digest not of the sha256 form.
|
|
func EncodeNodeWant(w NodeWant) ([]byte, error) {
|
|
if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil {
|
|
return nil, err
|
|
}
|
|
w.V = Version
|
|
return json.Marshal(w)
|
|
}
|
|
|
|
// DecodeNodeWant reads a node's own want; it refuses an unknown version (ErrUnknownVersion), a field a node's own want
|
|
// does not have (a fingerprint among them), and what EncodeNodeWant refuses.
|
|
func DecodeNodeWant(data []byte) (NodeWant, error) {
|
|
if err := checkVersion(data); err != nil {
|
|
return NodeWant{}, err
|
|
}
|
|
var w NodeWant
|
|
d := json.NewDecoder(bytes.NewReader(data))
|
|
d.DisallowUnknownFields()
|
|
if err := d.Decode(&w); err != nil {
|
|
return NodeWant{}, fmt.Errorf("delivery: a node's own want does not fit its shape: %w", err)
|
|
}
|
|
if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil {
|
|
return NodeWant{}, err
|
|
}
|
|
return w, nil
|
|
}
|
|
|
|
// EncodeHeld is h as JSON at Version; it refuses a node name a key refuses.
|
|
func EncodeHeld(h Held) ([]byte, error) {
|
|
if err := h.check(); err != nil {
|
|
return nil, err
|
|
}
|
|
h.V = Version
|
|
return json.Marshal(h)
|
|
}
|
|
|
|
// DecodeHeld reads a hold; it refuses an unknown version (ErrUnknownVersion) and what EncodeHeld refuses.
|
|
func DecodeHeld(data []byte) (Held, error) {
|
|
var h Held
|
|
if err := decode(data, &h); err != nil {
|
|
return Held{}, err
|
|
}
|
|
if err := h.check(); err != nil {
|
|
return Held{}, err
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// SameHeld says whether two holds are the same value.
|
|
func SameHeld(a, b Held) bool { return a == b }
|
|
|
|
// EncodeDifference is d as JSON at Version; it refuses a difference with neither a rule nor unexplained, one with both,
|
|
// a rule outside the six, and a want equal to the send.
|
|
func EncodeDifference(d Difference) ([]byte, error) {
|
|
if err := d.check(); err != nil {
|
|
return nil, err
|
|
}
|
|
d.V = Version
|
|
return json.Marshal(d)
|
|
}
|
|
|
|
// DecodeDifference reads a difference; it refuses an unknown version (ErrUnknownVersion) and what EncodeDifference
|
|
// refuses.
|
|
func DecodeDifference(data []byte) (Difference, error) {
|
|
var d Difference
|
|
if err := decode(data, &d); err != nil {
|
|
return Difference{}, err
|
|
}
|
|
if err := d.check(); err != nil {
|
|
return Difference{}, err
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// SameDifference says whether two differences are the same value.
|
|
func SameDifference(a, b Difference) bool { return reflect.DeepEqual(a, b) }
|
|
|
|
// The assigner's composition check (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, ADR 0482 clauses 3 and 9): for
|
|
// each group the checker judges, the assigner composes the machines the change touches and writes its verdict under
|
|
// the checker's key of that check, `<escaped group>.<heads hash>` (CheckerVerdictKey's form), in its bucket
|
|
// compositions. The checker watches that key and reads it into its verdict's composition (Composition); a key not yet
|
|
// written is a composition still running.
|
|
const (
|
|
// AssignerCompositionsBucket is the assigner's bucket of composition verdicts.
|
|
AssignerCompositionsBucket = AssignerModule + "_compositions"
|
|
// AssignerCompositionsHistory is the revisions the bucket keeps per key.
|
|
AssignerCompositionsHistory = 5
|
|
// CompositionWaits is the state of a composition that waits for what it composes from (ADR 0446 clause 3: the
|
|
// indexer holds no candidates for the group yet), its Words saying what; it is no verdict.
|
|
CompositionWaits = "waits"
|
|
// CompositionRebuildWarning is the most modules a merge may rebuild before the composition warns (ADR 0446 clause 1).
|
|
CompositionRebuildWarning = 12
|
|
)
|
|
|
|
// CompositionKey is the key, in AssignerCompositionsBucket, of the composition of a group's check at its heads: the
|
|
// checker's verdict key of the same check (CheckerVerdictKey).
|
|
func CompositionKey(group, headsHash string) (string, error) {
|
|
return CheckerVerdictKey(group, headsHash)
|
|
}
|
|
|
|
// ParseCompositionKey is the inverse of CompositionKey; it refuses a want, held or difference key, naming it.
|
|
func ParseCompositionKey(key string) (group, headsHash string, err error) {
|
|
if group, headsHash, err = ParseCheckerVerdictKey(key); err != nil {
|
|
return "", "", fmt.Errorf("delivery: %q is not a composition key, <group>.<heads hash>: %w", key, err)
|
|
}
|
|
return group, headsHash, nil
|
|
}
|
|
|
|
// CompositionVerdict is the value of a composition key: the group and its heads hash, the State (pass, fail or waits),
|
|
// the Words of a refusal or of what it waits for, the mesh-sdk commit it composed with (the assigner's own, or the
|
|
// group's mesh-sdk member's head, ADR 0482 clause 3), the machines composed, the first machine that failed, the
|
|
// warnings that fail nothing (more than twelve modules rebuilt; what was already broken before the change, ADR 0446
|
|
// clause 1), and the time it took in integer milliseconds.
|
|
type CompositionVerdict struct {
|
|
V int `json:"v"`
|
|
Group string `json:"group"`
|
|
Heads string `json:"heads"`
|
|
State string `json:"state"`
|
|
Words string `json:"words,omitempty"`
|
|
SDK string `json:"sdk,omitempty"`
|
|
Machines []string `json:"machines,omitempty"`
|
|
Machine string `json:"machine,omitempty"`
|
|
Warnings []string `json:"warnings,omitempty"`
|
|
Took int64 `json:"took,omitempty"`
|
|
}
|
|
|
|
func (c CompositionVerdict) check() error {
|
|
if c.Group == "" {
|
|
return fmt.Errorf("delivery: a composition names no group")
|
|
}
|
|
if !isHex12(c.Heads) {
|
|
return fmt.Errorf("delivery: the composition's heads hash %q is not 12 lowercase hex digits", c.Heads)
|
|
}
|
|
if c.Took < 0 {
|
|
return fmt.Errorf("delivery: the composition's time is negative")
|
|
}
|
|
switch c.State {
|
|
case CompositionPass:
|
|
if c.SDK == "" {
|
|
return fmt.Errorf("delivery: a composition that passed names no mesh-sdk commit")
|
|
}
|
|
if c.Machine != "" || c.Words != "" {
|
|
return fmt.Errorf("delivery: a composition that passed names no failure")
|
|
}
|
|
case CompositionFail:
|
|
if c.Words == "" {
|
|
return fmt.Errorf("delivery: a composition that failed gives no words")
|
|
}
|
|
if c.SDK == "" {
|
|
return fmt.Errorf("delivery: a composition that failed names no mesh-sdk commit")
|
|
}
|
|
case CompositionWaits:
|
|
if c.Words == "" {
|
|
return fmt.Errorf("delivery: a composition that waits does not say for what")
|
|
}
|
|
if c.Machine != "" || len(c.Machines) > 0 {
|
|
return fmt.Errorf("delivery: a composition that waits composed no machine")
|
|
}
|
|
default:
|
|
return fmt.Errorf("delivery: the composition state %q is not %s, %s or %s", c.State, CompositionPass, CompositionFail, CompositionWaits)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Composition is what the checker writes into its verdict from c: nil while c waits (the check is still running).
|
|
func (c CompositionVerdict) Composition() *Composition {
|
|
if c.State == CompositionWaits {
|
|
return nil
|
|
}
|
|
return &Composition{State: c.State, Words: c.Words, SDK: c.SDK, Machines: c.Machines}
|
|
}
|
|
|
|
// EncodeCompositionVerdict is c as JSON at Version; it refuses an unknown state, a pass or fail naming no mesh-sdk
|
|
// commit, a fail without words, a pass naming a failure, and a wait that composed a machine.
|
|
func EncodeCompositionVerdict(c CompositionVerdict) ([]byte, error) {
|
|
if err := c.check(); err != nil {
|
|
return nil, err
|
|
}
|
|
c.V = Version
|
|
return json.Marshal(c)
|
|
}
|
|
|
|
// DecodeCompositionVerdict reads a composition; it refuses an unknown version (ErrUnknownVersion) and what
|
|
// EncodeCompositionVerdict refuses.
|
|
func DecodeCompositionVerdict(data []byte) (CompositionVerdict, error) {
|
|
var c CompositionVerdict
|
|
if err := decode(data, &c); err != nil {
|
|
return CompositionVerdict{}, err
|
|
}
|
|
if err := c.check(); err != nil {
|
|
return CompositionVerdict{}, err
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// SameCompositionVerdict says whether two compositions are the same value.
|
|
func SameCompositionVerdict(a, b CompositionVerdict) bool { return reflect.DeepEqual(a, b) }
|