// Package outside is the assigner's wait on an outside cause (design 47 G §706, ADR 0677): a moved module's failing health // check on a machine whose open conditions name a cause outside the delivery is held as a wait, judged again on every change // that can end it, and ended by itself or by the operator. The package holds the rules and the kept record; the assigner's // process reads the conditions, says the events and writes the asks. package outside import ( "encoding/json" "fmt" "slices" "strings" "time" ) // The ways a wait ends, as `wait-ended` says them (§706 *How it ends*). const ( HowHealthy = "healthy" HowCleared = "cleared" HowOwnFailure = "own-failure" HowGoOn = "go-on" HowFailed = "failed" HowReplaced = "replaced" // HowRejudge is a release by hand that asks to judge the wait again now; it ends nothing. HowRejudge = "rejudge" ) // OnUnread is what a wait stands on while the keeper does not answer. const OnUnread = "conditions-unread" // The defaults of the settings (§633): `outside-wait-minutes` and `outside-settle-seconds`. const ( DefaultBound = 30 * time.Minute DefaultSettle = 90 * time.Second ) // Cond is an open condition as the keeper's verb `conditions` answers it: its key and the subject's machine and `also`. type Cond struct { Key string Machine string Also []string } // ParseConditions reads the keeper's answer of `conditions`. func ParseConditions(answer []byte) ([]Cond, error) { var a struct { Conditions []struct { Key string `json:"key"` Subject struct { Machine string `json:"machine"` Also []string `json:"also"` } `json:"subject"` } `json:"conditions"` } if err := json.Unmarshal(answer, &a); err != nil { return nil, fmt.Errorf("outside: the keeper's conditions do not read: %w", err) } out := make([]Cond, 0, len(a.Conditions)) for _, c := range a.Conditions { out = append(out, Cond{Key: c.Key, Machine: c.Subject.Machine, Also: c.Subject.Also}) } return out, nil } // Cause is an outside cause open for a machine: the condition's key and what a wait says it stands `on`. type Cause struct{ Key, On string } // Causes are the outside causes open for machine m among conds, sorted by key (§706 *What counts as an outside cause*): // `machine.M..rewritten` or `machine.M.rewritten` unless the delivery moved to M (moved), and // `machine..unreachable`, `.network` or `.silent` of another machine whose `also` lists M. No other condition is one. func Causes(conds []Cond, m string, moved func(owner string) bool) []Cause { var out []Cause for _, c := range conds { p := strings.Split(c.Key, ".") if len(p) < 3 || p[0] != "machine" { continue } switch { case len(p) == 3 && p[1] == m && p[2] == "rewritten": out = append(out, Cause{c.Key, "rewritten"}) case len(p) == 4 && p[1] == m && p[3] == "rewritten" && (moved == nil || !moved(p[2])): out = append(out, Cause{c.Key, "rewritten:" + p[2]}) case len(p) == 3 && p[1] != m && slices.Contains([]string{"unreachable", "network", "silent"}, p[2]) && slices.Contains(c.Also, m): out = append(out, Cause{c.Key, p[2] + ":" + p[1]}) } } slices.SortFunc(out, func(a, b Cause) int { return strings.Compare(a.Key, b.Key) }) return out } // OwnNetworkKey is the key of a machine's own open network condition, which holds the machine as a whole (§706 *M's own // network*). func OwnNetworkKey(m string) string { return "machine." + m + ".network" } // OwnNetwork are the machines whose own network condition is open among conds. func OwnNetwork(conds []Cond) map[string]bool { out := map[string]bool{} for _, c := range conds { if p := strings.Split(c.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" && c.Key == OwnNetworkKey(p[1]) { out[p[1]] = true } } return out } // Wait is one outside wait as the assigner keeps it: the move of Module at FP on Node, or its settle (Settling). Decided is // what ended it for the rest of that fingerprint on that node: `go-on` (counted healthy by the operator's word) or `failed` // (the delivery's failure), so a check that goes on failing is not waited on again. type Wait struct { V int `json:"v"` // Step is set only on a record of an acting wait (internal/acting), which this package never reads as its own. Step string `json:"step,omitempty"` Node string `json:"node"` Module string `json:"module"` FP string `json:"fp"` On string `json:"on,omitempty"` Cause string `json:"cause,omitempty"` Since time.Time `json:"since"` ClockFrom time.Time `json:"clock-from"` Bound string `json:"bound,omitempty"` // Settling: no outside cause was open at the failing check; the judgement is held until SettleEnds. Settling bool `json:"settling,omitempty"` SettleEnds time.Time `json:"settle-ends,omitempty"` // Ask is the id of the asking made at the bound, and State where the wait stands in words (settling, waiting, asked). Ask string `json:"ask,omitempty"` State string `json:"state,omitempty"` // Release is a release by hand not taken up yet: go-on, failed or rejudge, by By and why Why. Release string `json:"release,omitempty"` By string `json:"by,omitempty"` Why string `json:"why,omitempty"` // Decided is `go-on` or `failed` once the wait ended so. Decided string `json:"decided,omitempty"` } // Key is the key of a wait in the state: `..`. func Key(node, module, hex string) string { return node + "." + module + "." + hex } // Encode is w as JSON. func Encode(w Wait) ([]byte, error) { w.V = 1; return json.Marshal(w) } // Decode reads a kept wait. func Decode(raw []byte) (Wait, error) { var w Wait if err := json.Unmarshal(raw, &w); err != nil { return w, err } if w.V != 1 || w.Step != "" || w.Node == "" || w.Module == "" { return w, fmt.Errorf("outside: %q is not a kept outside wait", raw) } return w, nil } // Standing says a wait is said (`wait-held`) and not only settling, nor decided. func (w Wait) Standing() bool { return !w.Settling && w.Decided == "" } // Facts are what a judging reads: the module's check on the node at the wait's fingerprint (Failing, Healthy), the keeper's // conditions as read now (Read false: the kept call is unanswered), whether the module holds the seat mesh-conditions itself, // and the settings. type Facts struct { Now time.Time Failing bool Healthy bool Read bool Conds []Cond Node string // Moved says the delivery moved the module to the node. Moved func(owner string) bool // CauseCleared: the condition the wait stands on was heard cleared (the keeper's event), so a healthy module ends `cleared`. CauseCleared bool // KeeperSelf: the module holds the seat mesh-conditions, which cannot answer for its own failing check. KeeperSelf bool Bound time.Duration Settle time.Duration } // Decision is what a judging comes to. type Decision struct { // Wait is the record to keep: nil when none stands (none is kept; a decided one is kept with Decided). Wait *Wait // Held says wait-held is to be said (a new wait, or a new `on`); End is the `how` of a wait-ended to say, "" for none. Held bool End string // Own: the failure is the delivery's (the rules hold it as the canary rule has it). Counted: the module counts healthy // there by the operator's word. Waiting: the move waits (the rules see no failure, and the canary holds the rest). Own, Counted, Waiting bool // Ask: the bound has run from clock-from, so the operator is asked. Ask bool // Due is when the judging is due again by itself (the settle's end); zero for none. Due time.Time } // Judge is one judging of the check at the node (§706). prev is the kept wait of that module at that fingerprint, nil for none. func Judge(prev *Wait, node, module, fp string, f Facts) Decision { if prev != nil && prev.Decided != "" { // Decided by the operator: it holds for this fingerprint until the module runs healthy. if f.Healthy { return Decision{} } d := Decision{Wait: prev, Own: prev.Decided == HowFailed, Counted: prev.Decided == HowGoOn} return d } if prev != nil && prev.Release != "" && prev.Release != HowRejudge { how := prev.Release w := *prev w.Release, w.Decided, w.Settling, w.Ask = "", how, false, "" d := Decision{Wait: &w, Own: how == HowFailed, Counted: how == HowGoOn} if prev.Standing() { d.End = how } return d } if f.Healthy { if prev == nil { return Decision{} } d := Decision{} if prev.Standing() { d.End = HowHealthy if prev.Cause != "" && (f.CauseCleared || (f.Read && !openKey(f.Conds, prev.Cause))) { d.End = HowCleared } } return d } if !f.Failing { // still moving: nothing is judged, what stands stands if prev == nil { return Decision{} } return Decision{Wait: prev, Waiting: prev.Standing()} } if f.KeeperSelf { d := Decision{Own: true} if prev != nil && prev.Standing() { d.End = HowOwnFailure } return d } next := Wait{Node: node, Module: module, FP: fp, Bound: f.Bound.String(), Since: f.Now, ClockFrom: f.Now} standing := prev != nil && prev.Standing() if standing { next = *prev next.Release = "" } if !f.Read { // The keeper does not answer. A judging that has only just asked holds, as the settle does; once the settle has run // from the failing check with no answer, the move waits on conditions-unread (§706 *When the keeper does not answer*). if standing { return stand(prev, *prev, f) // a wait that stands keeps its cause until the keeper answers } { if prev != nil && prev.Settling { next = *prev } else { next.Settling, next.SettleEnds, next.State = true, f.Now.Add(f.Settle), "settling" } if f.Now.Before(next.SettleEnds) { return Decision{Wait: &next, Waiting: true, Due: next.SettleEnds} } next.Settling, next.SettleEnds = false, time.Time{} } next.On, next.Cause, next.Settling, next.State = OnUnread, "", false, "waiting" return stand(prev, next, f) } causes := Causes(f.Conds, node, f.Moved) if len(causes) > 0 { c := causes[0] if standing { for _, k := range causes { if k.Key == prev.Cause { c = k } } } next.On, next.Cause, next.Settling, next.SettleEnds, next.State = c.On, c.Key, false, time.Time{}, "waiting" return stand(prev, next, f) } if standing { // judged again, failing, no outside cause open: the delivery's failure (after the settle the check has long had) w := *prev w.Decided, w.Release, w.Ask = HowOwnFailure, "", "" return Decision{Wait: &w, End: HowOwnFailure, Own: true} } // no outside cause open: settle from the failing check, judged again when it ends if prev != nil && prev.Settling { if !f.Now.Before(prev.SettleEnds) { w := *prev w.Settling, w.Decided = false, HowOwnFailure return Decision{Wait: &w, Own: true} } return Decision{Wait: prev, Waiting: true, Due: prev.SettleEnds} } next.Settling, next.SettleEnds, next.State, next.On, next.Cause = true, f.Now.Add(f.Settle), "settling", "", "" return Decision{Wait: &next, Waiting: true, Due: next.SettleEnds} } // stand is the decision of a wait that stands on next.On: said when it is new or stands on another cause, its clock kept. func stand(prev *Wait, next Wait, f Facts) Decision { d := Decision{Wait: &next, Waiting: true} if prev == nil || !prev.Standing() || prev.On != next.On { d.Held = true } if prev != nil && prev.Standing() { next.Ask = prev.Ask } if f.Bound > 0 && !f.Now.Before(next.ClockFrom.Add(f.Bound)) { d.Ask = true next.State = "asked" } d.Wait = &next return d } func openKey(conds []Cond, key string) bool { return slices.ContainsFunc(conds, func(c Cond) bool { return c.Key == key }) }