mesh-controller/cmd/mesh-controller/busy.go

package main

import (
	"fmt"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
)

// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
//
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
// its gate starts as before.

// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
var busyGateSlack = 5 * time.Minute

// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
// controller's clock from when it first read the wait.
const busyEngineBound = 15 * time.Minute

// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
// machine nor a wrong word from it holds a walk longer.
func busyGateEnds(b *inventory.GateBusy) time.Time {
	until := b.Seen.Add(busyEngineBound)
	if !b.Until.IsZero() && b.Until.Before(until) {
		until = b.Until
	}
	return until.Add(busyGateSlack)
}

// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
// gate's Since to its end. Pure but for the gate it is handed.
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
	if g.Busy != nil && g.Busy.Ended != nil {
		return false
	}
	for _, j := range pairs {
		b, waits := busyOn(j, f)
		if !waits {
			continue
		}
		if g.Busy == nil {
			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
				Seen: now}
		}
		if g.Busy.Seen.IsZero() {
			g.Busy.Seen = now
		}
		g.Busy.Until, g.Busy.Said = b.Until, b.Said
		if now.After(busyGateEnds(g.Busy)) {
			// The engine said it would have replaced it by now: the judging starts, and finds the module not
			// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
			end := now
			g.Busy.Ended, g.Busy.PastBound = &end, true
			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
			return false
		}
		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
		g.Read(now, false, g.Busy.Words())
		return true
	}
	if g.Busy == nil {
		return false
	}
	// Replaced: the judging starts at the report that said so.
	end := now
	if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
		end = r.At.UTC()
	}
	g.Busy.Ended = &end
	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
	return false
}

// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
// running because it said busy.
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
	r, said := f.reports[j.node]
	if !said || r.At == nil || !f.reportedOn(j.node, r) {
		return inventory.BusyWait{}, false
	}
	return inventory.WaitingFor(r.Busy, j.module)
}

// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
func busyStill(j judged, f gateFacts) (string, bool) {
	b, waits := busyOn(j, f)
	if !waits {
		return "", false
	}
	return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
		j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
}