package apply import ( "context" "fmt" "strings" "time" "unicode/utf8" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335). // // **Asked, never waited on.** The apply asks the container once, and while it says busy leaves it running // as it is and goes on with everything else of the declaration: the machine's lock is held for one ask, // never for the wait. The wait is carried between passes in the container's record (store.Applied's // BusySince), so the next pass — the node-engine asks again sooner than its reconcile while anything waits // — asks again, and so does a node-engine started again in between: the bound runs from the first apply // that would have replaced the container, whoever applied it. // waitsBusy is the action of a container left running because it said busy (novox/hq ADR 0335 §2). Its own // word rather than "unchanged": its declaration moved, and the machine is not yet what was declared. const waitsBusy = "waits-busy" // What a report says of a busy container (novox/hq ADR 0335 §2, §3). const ( // BusyWaiting is a container left running because it said busy, waiting to be replaced. BusyWaiting = "waiting" // BusyReplacedIdle is a container replaced on the first pass it said idle, after it had waited. BusyReplacedIdle = "replaced-when-idle" // BusyReplacedAtBound is a container still busy at the bound, replaced anyway. BusyReplacedAtBound = "replaced-at-bound" ) // BusyWait is one container a busy command held, as a report says it. type BusyWait struct { // ID is the resource's id (`.`), Container its runtime name. ID string `json:"id"` Container string `json:"container"` // State is BusyWaiting, BusyReplacedIdle or BusyReplacedAtBound. State string `json:"state"` // Since is when the first apply that would have replaced it ran; Until when the bound replaces it. Since time.Time `json:"since"` Until time.Time `json:"until"` // Said is the last output its busy command gave. Said string `json:"said,omitempty"` } // Words is the wait in a sentence, for an outcome's detail and the console. func (b BusyWait) Words() string { since := b.Since.UTC().Format(time.RFC3339) said := "" if b.Said != "" { said = "; its busy command last said: " + b.Said } switch b.State { case BusyReplacedAtBound: return fmt.Sprintf("replaced while still busy: it said busy since %s and the bound of %s ran out "+ "(novox/hq ADR 0335 §3)%s", since, declaration.BusyBound, said) case BusyReplacedIdle: return fmt.Sprintf("replaced once it said idle, after waiting since %s (novox/hq ADR 0335 §2)%s", since, said) } return fmt.Sprintf("busy: left running and replaced on the first pass it says idle, or at %s at the latest; "+ "waiting since %s (novox/hq ADR 0335 §2)%s", b.Until.UTC().Format(time.RFC3339), since, said) } // owedSinceBusy is the reason a container still owes the replacement it was left running for, or empty: its // record keeps when its busy wait began, under the same name. func owedSinceBusy(r *declaration.Container, previous store.Applied) string { if previous.BusySince == nil || previous.Target != r.Name { return "" } return "the replacement owed since its busy wait began at " + previous.BusySince.UTC().Format(time.RFC3339) } // busyClock is the clock the bound is read on. A test moves it. var busyClock = time.Now // busySaidMost is how much of a busy command's output a report carries: its last lines, enough to say what // it held. const busySaidMost = 400 // askBeforeReplacing asks a running container that declares busy whether it may be replaced now, and says // in out what came of it. True when it waits: out is then the whole outcome of this pass. False when the // caller replaces it — idle, or past the bound — with out carrying the wait it ended, if there was one. func askBeforeReplacing(ctx context.Context, r *declaration.Container, run Runner, cri string, previous store.Applied, out *Outcome) bool { now := busyClock().UTC() busy, said := askBusy(ctx, r, run, cri) var since time.Time waited := previous.BusySince != nil && previous.Target == r.Name if waited { since = previous.BusySince.UTC() // A wait kept from a clock that has since gone back reads as starting now, never as later: the bound // holds from no later than this pass. if since.After(now) { since = now } } wait := BusyWait{ID: r.ID, Container: r.Name, Since: since, Said: said} switch { case !busy && !waited: // Idle at the first ask: replaced at once, as a container that declares nothing is. return false case !busy: wait.State, wait.Until = BusyReplacedIdle, since.Add(declaration.BusyBound) out.busy = &wait return false } if !waited { since = now wait.Since = now } wait.Until = since.Add(declaration.BusyBound) if !now.Before(wait.Until) { wait.State = BusyReplacedAtBound out.busy = &wait return false } wait.State = BusyWaiting out.Action, out.Detail = waitsBusy, wait.Words() out.busySince, out.busy = &since, &wait return true } // askBusy runs a container's busy command inside it, by its shell, as an exec health check is run: busy is // an exit of 0 and nothing else. An ask that could not be made, or did not answer within its timeout, is // not busy: the bound is what keeps a container whose command never answers from being held. func askBusy(ctx context.Context, r *declaration.Container, run Runner, cri string) (bool, string) { ask, cancel := context.WithTimeout(ctx, r.Busy.Within()) defer cancel() output, err := run(ask, cri, "exec", r.Name, "/bin/sh", "-c", r.Busy.Command) said := strings.TrimSpace(output) if err != nil && said == "" { said = err.Error() } return err == nil, lastOf(said, busySaidMost) } // lastOf is the end of a text, at most n bytes, on one line. func lastOf(s string, n int) string { s = strings.Join(strings.Fields(s), " ") if len(s) <= n { return s } from := len(s) - n for from < len(s) && !utf8.RuneStart(s[from]) { from++ } return "…" + s[from:] }