package main import ( "fmt" "time" "github.com/novox/mesh-controller/internal/inventory" ) // A first-node gate waits for a busy container (novox/hq ADR 0335 §4). // // When a machine reports, on what the gate sent it, that a container of a judged module was left running // because it said busy, the judging has not started: the build is not yet what runs there. The walk says it // waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its // Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded // at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped // reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and // its gate starts as before. // busyGateSlack is how long past the engine's bound the controller still waits for the report that says the // container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here. var busyGateSlack = 5 * time.Minute // busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this // controller's clock from when it first read the wait. const busyEngineBound = 15 * time.Minute // busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of // it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the // machine nor a wrong word from it holds a walk longer. func busyGateEnds(b *inventory.GateBusy) time.Time { until := b.Seen.Add(busyEngineBound) if !b.Until.IsZero() && b.Until.Before(until) { until = b.Until } return until.Add(busyGateSlack) } // busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges: // true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the // gate's Since to its end. Pure but for the gate it is handed. func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool { if g.Busy != nil && g.Busy.Ended != nil { return false } for _, j := range pairs { b, waits := busyOn(j, f) if !waits { continue } if g.Busy == nil { g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since, Seen: now} } if g.Busy.Seen.IsZero() { g.Busy.Seen = now } g.Busy.Until, g.Busy.Said = b.Until, b.Said if now.After(busyGateEnds(g.Busy)) { // The engine said it would have replaced it by now: the judging starts, and finds the module not // yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound. end := now g.Busy.Ended, g.Busy.PastBound = &end, true g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil return false } g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil g.Read(now, false, g.Busy.Words()) return true } if g.Busy == nil { return false } // Replaced: the judging starts at the report that said so. end := now if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) { end = r.At.UTC() } g.Busy.Ended = &end g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil return false } // busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left // running because it said busy. func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) { r, said := f.reports[j.node] if !said || r.At == nil || !f.reportedOn(j.node, r) { return inventory.BusyWait{}, false } return inventory.WaitingFor(r.Busy, j.module) } // busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting // for it: not yet what was sent, so not healthy, whatever its health says of the container still running. func busyStill(j judged, f gateFacts) (string, bool) { b, waits := busyOn(j, f) if !waits { return "", false } return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s", j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true }