mesh-delivery/modules/mesh-delivery-assigner/internal/outside/outside.go

// Package outside is the assigner's wait on an outside cause (design 47 G §706, ADR 0677): a moved module's failing health
// check on a machine whose open conditions name a cause outside the delivery is held as a wait, judged again on every change
// that can end it, and ended by itself or by the operator. The package holds the rules and the kept record; the assigner's
// process reads the conditions, says the events and writes the asks.
package outside

import (
	"encoding/json"
	"fmt"
	"slices"
	"strings"
	"time"
)

// The ways a wait ends, as `wait-ended` says them (§706 *How it ends*).
const (
	HowHealthy    = "healthy"
	HowCleared    = "cleared"
	HowOwnFailure = "own-failure"
	HowGoOn       = "go-on"
	HowFailed     = "failed"
	HowReplaced   = "replaced"
	// HowRejudge is a release by hand that asks to judge the wait again now; it ends nothing.
	HowRejudge = "rejudge"
)

// OnUnread is what a wait stands on while the keeper does not answer.
const OnUnread = "conditions-unread"

// The defaults of the settings (§633): `outside-wait-minutes` and `outside-settle-seconds`.
const (
	DefaultBound  = 30 * time.Minute
	DefaultSettle = 90 * time.Second
)

// Cond is an open condition as the keeper's verb `conditions` answers it: its key and the subject's machine and `also`.
type Cond struct {
	Key     string
	Machine string
	Also    []string
}

// ParseConditions reads the keeper's answer of `conditions`.
func ParseConditions(answer []byte) ([]Cond, error) {
	var a struct {
		Conditions []struct {
			Key     string `json:"key"`
			Subject struct {
				Machine string   `json:"machine"`
				Also    []string `json:"also"`
			} `json:"subject"`
		} `json:"conditions"`
	}
	if err := json.Unmarshal(answer, &a); err != nil {
		return nil, fmt.Errorf("outside: the keeper's conditions do not read: %w", err)
	}
	out := make([]Cond, 0, len(a.Conditions))
	for _, c := range a.Conditions {
		out = append(out, Cond{Key: c.Key, Machine: c.Subject.Machine, Also: c.Subject.Also})
	}
	return out, nil
}

// Cause is an outside cause open for a machine: the condition's key and what a wait says it stands `on`.
type Cause struct{ Key, On string }

// Causes are the outside causes open for machine m among conds, sorted by key (§706 *What counts as an outside cause*):
// `machine.M.<owner>.rewritten` or `machine.M.rewritten` unless the delivery moved <owner> to M (moved), and
// `machine.<x>.unreachable`, `.network` or `.silent` of another machine whose `also` lists M. No other condition is one.
func Causes(conds []Cond, m string, moved func(owner string) bool) []Cause {
	var out []Cause
	for _, c := range conds {
		p := strings.Split(c.Key, ".")
		if len(p) < 3 || p[0] != "machine" {
			continue
		}
		switch {
		case len(p) == 3 && p[1] == m && p[2] == "rewritten":
			out = append(out, Cause{c.Key, "rewritten"})
		case len(p) == 4 && p[1] == m && p[3] == "rewritten" && (moved == nil || !moved(p[2])):
			out = append(out, Cause{c.Key, "rewritten:" + p[2]})
		case len(p) == 3 && p[1] != m && slices.Contains([]string{"unreachable", "network", "silent"}, p[2]) && slices.Contains(c.Also, m):
			out = append(out, Cause{c.Key, p[2] + ":" + p[1]})
		}
	}
	slices.SortFunc(out, func(a, b Cause) int { return strings.Compare(a.Key, b.Key) })
	return out
}

// OwnNetworkKey is the key of a machine's own open network condition, which holds the machine as a whole (§706 *M's own
// network*).
func OwnNetworkKey(m string) string { return "machine." + m + ".network" }

// OwnNetwork are the machines whose own network condition is open among conds.
func OwnNetwork(conds []Cond) map[string]bool {
	out := map[string]bool{}
	for _, c := range conds {
		if p := strings.Split(c.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" && c.Key == OwnNetworkKey(p[1]) {
			out[p[1]] = true
		}
	}
	return out
}

// Wait is one outside wait as the assigner keeps it: the move of Module at FP on Node, or its settle (Settling). Decided is
// what ended it for the rest of that fingerprint on that node: `go-on` (counted healthy by the operator's word) or `failed`
// (the delivery's failure), so a check that goes on failing is not waited on again.
type Wait struct {
	V int `json:"v"`
	// Step is set only on a record of an acting wait (internal/acting), which this package never reads as its own.
	Step      string    `json:"step,omitempty"`
	Node      string    `json:"node"`
	Module    string    `json:"module"`
	FP        string    `json:"fp"`
	On        string    `json:"on,omitempty"`
	Cause     string    `json:"cause,omitempty"`
	Since     time.Time `json:"since"`
	ClockFrom time.Time `json:"clock-from"`
	Bound     string    `json:"bound,omitempty"`
	// Settling: no outside cause was open at the failing check; the judgement is held until SettleEnds.
	Settling   bool      `json:"settling,omitempty"`
	SettleEnds time.Time `json:"settle-ends,omitempty"`
	// Ask is the id of the asking made at the bound, and State where the wait stands in words (settling, waiting, asked).
	Ask   string `json:"ask,omitempty"`
	State string `json:"state,omitempty"`
	// Release is a release by hand not taken up yet: go-on, failed or rejudge, by By and why Why.
	Release string `json:"release,omitempty"`
	By      string `json:"by,omitempty"`
	Why     string `json:"why,omitempty"`
	// Decided is `go-on` or `failed` once the wait ended so.
	Decided string `json:"decided,omitempty"`
}

// Key is the key of a wait in the state: `<node>.<module>.<fingerprint hex>`.
func Key(node, module, hex string) string { return node + "." + module + "." + hex }

// Encode is w as JSON.
func Encode(w Wait) ([]byte, error) { w.V = 1; return json.Marshal(w) }

// Decode reads a kept wait.
func Decode(raw []byte) (Wait, error) {
	var w Wait
	if err := json.Unmarshal(raw, &w); err != nil {
		return w, err
	}
	if w.V != 1 || w.Step != "" || w.Node == "" || w.Module == "" {
		return w, fmt.Errorf("outside: %q is not a kept outside wait", raw)
	}
	return w, nil
}

// Standing says a wait is said (`wait-held`) and not only settling, nor decided.
func (w Wait) Standing() bool { return !w.Settling && w.Decided == "" }

// Facts are what a judging reads: the module's check on the node at the wait's fingerprint (Failing, Healthy), the keeper's
// conditions as read now (Read false: the kept call is unanswered), whether the module holds the seat mesh-conditions itself,
// and the settings.
type Facts struct {
	Now     time.Time
	Failing bool
	Healthy bool
	Read    bool
	Conds   []Cond
	Node    string
	// Moved says the delivery moved the module <owner> to the node.
	Moved func(owner string) bool
	// CauseCleared: the condition the wait stands on was heard cleared (the keeper's event), so a healthy module ends `cleared`.
	CauseCleared bool
	// KeeperSelf: the module holds the seat mesh-conditions, which cannot answer for its own failing check.
	KeeperSelf bool
	Bound      time.Duration
	Settle     time.Duration
}

// Decision is what a judging comes to.
type Decision struct {
	// Wait is the record to keep: nil when none stands (none is kept; a decided one is kept with Decided).
	Wait *Wait
	// Held says wait-held is to be said (a new wait, or a new `on`); End is the `how` of a wait-ended to say, "" for none.
	Held bool
	End  string
	// Own: the failure is the delivery's (the rules hold it as the canary rule has it). Counted: the module counts healthy
	// there by the operator's word. Waiting: the move waits (the rules see no failure, and the canary holds the rest).
	Own, Counted, Waiting bool
	// Ask: the bound has run from clock-from, so the operator is asked.
	Ask bool
	// Due is when the judging is due again by itself (the settle's end); zero for none.
	Due time.Time
}

// Judge is one judging of the check at the node (§706). prev is the kept wait of that module at that fingerprint, nil for none.
func Judge(prev *Wait, node, module, fp string, f Facts) Decision {
	if prev != nil && prev.Decided != "" {
		// Decided by the operator: it holds for this fingerprint until the module runs healthy.
		if f.Healthy {
			return Decision{}
		}
		d := Decision{Wait: prev, Own: prev.Decided == HowFailed, Counted: prev.Decided == HowGoOn}
		return d
	}
	if prev != nil && prev.Release != "" && prev.Release != HowRejudge {
		how := prev.Release
		w := *prev
		w.Release, w.Decided, w.Settling, w.Ask = "", how, false, ""
		d := Decision{Wait: &w, Own: how == HowFailed, Counted: how == HowGoOn}
		if prev.Standing() {
			d.End = how
		}
		return d
	}
	if f.Healthy {
		if prev == nil {
			return Decision{}
		}
		d := Decision{}
		if prev.Standing() {
			d.End = HowHealthy
			if prev.Cause != "" && (f.CauseCleared || (f.Read && !openKey(f.Conds, prev.Cause))) {
				d.End = HowCleared
			}
		}
		return d
	}
	if !f.Failing {
		// still moving: nothing is judged, what stands stands
		if prev == nil {
			return Decision{}
		}
		return Decision{Wait: prev, Waiting: prev.Standing()}
	}
	if f.KeeperSelf {
		d := Decision{Own: true}
		if prev != nil && prev.Standing() {
			d.End = HowOwnFailure
		}
		return d
	}
	next := Wait{Node: node, Module: module, FP: fp, Bound: f.Bound.String(), Since: f.Now, ClockFrom: f.Now}
	standing := prev != nil && prev.Standing()
	if standing {
		next = *prev
		next.Release = ""
	}
	if !f.Read {
		// The keeper does not answer. A judging that has only just asked holds, as the settle does; once the settle has run
		// from the failing check with no answer, the move waits on conditions-unread (§706 *When the keeper does not answer*).
		if standing {
			return stand(prev, *prev, f) // a wait that stands keeps its cause until the keeper answers
		}
		{
			if prev != nil && prev.Settling {
				next = *prev
			} else {
				next.Settling, next.SettleEnds, next.State = true, f.Now.Add(f.Settle), "settling"
			}
			if f.Now.Before(next.SettleEnds) {
				return Decision{Wait: &next, Waiting: true, Due: next.SettleEnds}
			}
			next.Settling, next.SettleEnds = false, time.Time{}
		}
		next.On, next.Cause, next.Settling, next.State = OnUnread, "", false, "waiting"
		return stand(prev, next, f)
	}
	causes := Causes(f.Conds, node, f.Moved)
	if len(causes) > 0 {
		c := causes[0]
		if standing {
			for _, k := range causes {
				if k.Key == prev.Cause {
					c = k
				}
			}
		}
		next.On, next.Cause, next.Settling, next.SettleEnds, next.State = c.On, c.Key, false, time.Time{}, "waiting"
		return stand(prev, next, f)
	}
	if standing {
		// judged again, failing, no outside cause open: the delivery's failure (after the settle the check has long had)
		w := *prev
		w.Decided, w.Release, w.Ask = HowOwnFailure, "", ""
		return Decision{Wait: &w, End: HowOwnFailure, Own: true}
	}
	// no outside cause open: settle from the failing check, judged again when it ends
	if prev != nil && prev.Settling {
		if !f.Now.Before(prev.SettleEnds) {
			w := *prev
			w.Settling, w.Decided = false, HowOwnFailure
			return Decision{Wait: &w, Own: true}
		}
		return Decision{Wait: prev, Waiting: true, Due: prev.SettleEnds}
	}
	next.Settling, next.SettleEnds, next.State, next.On, next.Cause = true, f.Now.Add(f.Settle), "settling", "", ""
	return Decision{Wait: &next, Waiting: true, Due: next.SettleEnds}
}

// stand is the decision of a wait that stands on next.On: said when it is new or stands on another cause, its clock kept.
func stand(prev *Wait, next Wait, f Facts) Decision {
	d := Decision{Wait: &next, Waiting: true}
	if prev == nil || !prev.Standing() || prev.On != next.On {
		d.Held = true
	}
	if prev != nil && prev.Standing() {
		next.Ask = prev.Ask
	}
	if f.Bound > 0 && !f.Now.Before(next.ClockFrom.Add(f.Bound)) {
		d.Ask = true
		next.State = "asked"
	}
	d.Wait = &next
	return d
}

func openKey(conds []Cond, key string) bool {
	return slices.ContainsFunc(conds, func(c Cond) bool { return c.Key == key })
}