The watchman keeps the watched branch each open take-in wait names (take-in.pending's branch) and times a head only while its branch has none; once the revision is taken in the head is timed again from its own move. The head on a branch whose branch-unprotected stands (§497's second sentence) is not built: nothing the watchman hears says it (the forge bridge writes that finding, no event). Builds: design 47 G §12, §75, §497 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
194 lines
7.2 KiB
Go
194 lines
7.2 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
)
|
|
|
|
// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the
|
|
// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the
|
|
// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head.
|
|
|
|
// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75).
|
|
const KindPipelineStands = "pipeline-stands"
|
|
|
|
// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five
|
|
// minutes).
|
|
const HeadBound = 5 * time.Minute
|
|
|
|
// headRow is the signals table's row of a head with no pipeline.
|
|
const headRow = "head-pipeline"
|
|
|
|
// head is a watched branch's head still waiting for its pipeline.
|
|
type head struct {
|
|
revision int64
|
|
at time.Time
|
|
}
|
|
|
|
// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline
|
|
// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5),
|
|
// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits.
|
|
func (w *Watchman) SourceMoved(body []byte) {
|
|
var m meshdelivery.SourceMovedEvent
|
|
if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 {
|
|
w.say("a source-moved does not read, left: %v", err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
id := headID(m.Branch)
|
|
w.unsignal(headRow, id)
|
|
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
|
if m.Group != "" {
|
|
delete(w.heads, m.Branch)
|
|
return
|
|
}
|
|
at := w.now()
|
|
if m.At != nil && !m.At.IsZero() {
|
|
at = *m.At
|
|
}
|
|
w.heads[m.Branch] = head{revision: m.Revision, at: at}
|
|
w.timeHead(m.Branch)
|
|
}
|
|
|
|
// timeHead times the head of a branch that waits for its pipeline, w.mu held: five minutes from its own time, unless the
|
|
// branch has a pending revision, whose take-in wait governs it (design 47 G §497) and which times it again once taken in.
|
|
func (w *Watchman) timeHead(branch string) {
|
|
h, ok := w.heads[branch]
|
|
if !ok || w.branchPending(branch) {
|
|
return
|
|
}
|
|
id := headID(branch)
|
|
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: h.at,
|
|
Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", branch, h.revision, stamp(h.at)),
|
|
Said: fmt.Sprintf("no pipeline in %s", HeadBound),
|
|
Headline: "A change has started no pipeline",
|
|
Explanation: "The head of the watched branch " + branch + " (source revision " + fmt.Sprint(h.revision) + ") moved " +
|
|
HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.",
|
|
Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"}
|
|
left := HeadBound - w.now().Sub(h.at)
|
|
if left > 0 {
|
|
w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f})
|
|
return
|
|
}
|
|
w.find(f)
|
|
}
|
|
|
|
// branchPending is whether an open take-in wait names the branch (design 47 G §497): its head is behind a pending revision.
|
|
func (w *Watchman) branchPending(branch string) bool {
|
|
for _, b := range w.pendingBranch {
|
|
if b == branch {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that
|
|
// source revision or a later one). A run on request is no head's pipeline.
|
|
func (w *Watchman) PipelineStarted(body []byte) {
|
|
var p meshpipelinerunner.PipelineStartedEvent
|
|
if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" {
|
|
w.say("a pipeline-started does not read, left: %v", err)
|
|
return
|
|
}
|
|
if p.Run != "" {
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
h, ok := w.heads[p.Branch]
|
|
if !ok || p.Source < h.revision {
|
|
return
|
|
}
|
|
delete(w.heads, p.Branch)
|
|
id := headID(p.Branch)
|
|
w.unsignal(headRow, id)
|
|
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
|
}
|
|
|
|
// headID is the finding's id of a branch's head.
|
|
func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") }
|
|
|
|
// KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86).
|
|
const KindBuildCapacityWaits = "build-capacity-waits"
|
|
|
|
// CapacityBound is how long a step may wait queued on a build seat's stream before the seat is a finding (design 47 G §86).
|
|
const CapacityBound = 60 * time.Minute
|
|
|
|
// capacityRow is the signals table's row of a build seat's queue.
|
|
const capacityRow = "build-capacity"
|
|
|
|
// StepEvent hears the runner's `step-started` and `step-ended` (design 47 G §86): a step said waiting for a build node is
|
|
// queued on its seat, owed no progress; it leaves the queue when it starts running or ends. A seat's finding is due when
|
|
// its oldest queued step has waited CapacityBound, and ends when that seat has none that old.
|
|
func (w *Watchman) StepEvent(body []byte) {
|
|
var e meshpipelinerunner.StepEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
|
w.say("a step event does not read, left: %v", err)
|
|
return
|
|
}
|
|
id := e.Pipeline + "/" + e.Result.Step
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
for seat, q := range w.queued {
|
|
if _, ok := q[id]; ok && (e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat != seat) {
|
|
delete(q, id)
|
|
w.capacity(seat)
|
|
}
|
|
}
|
|
if e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat == "" {
|
|
return
|
|
}
|
|
since := w.now()
|
|
switch {
|
|
case e.Result.Started != nil:
|
|
since = *e.Result.Started
|
|
case e.At != nil:
|
|
since = *e.At
|
|
}
|
|
seat := e.Result.Seat
|
|
if w.queued[seat] == nil {
|
|
w.queued[seat] = map[string]time.Time{}
|
|
}
|
|
if _, ok := w.queued[seat][id]; !ok {
|
|
w.queued[seat][id] = since
|
|
}
|
|
w.capacity(seat)
|
|
}
|
|
|
|
// capacity judges a build seat's queue again, w.mu held: one signal for its oldest queued step.
|
|
func (w *Watchman) capacity(seat string) {
|
|
w.unsignal(capacityRow, seat)
|
|
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
|
|
var oldest time.Time
|
|
for _, t := range w.queued[seat] {
|
|
if oldest.IsZero() || t.Before(oldest) {
|
|
oldest = t
|
|
}
|
|
}
|
|
if oldest.IsZero() {
|
|
delete(w.queued, seat)
|
|
w.unfind(key)
|
|
return
|
|
}
|
|
f := Finding{Scope: ScopeDelivery, ID: seat, Kind: KindBuildCapacityWaits, Severity: Warning, Source: capacityRow, Since: oldest,
|
|
Summary: fmt.Sprintf("the oldest step queued on %s has waited since %s", seat, stamp(oldest)),
|
|
Said: "queued for " + CapacityBound.String(),
|
|
Headline: "Builds wait for a build node",
|
|
Explanation: "The oldest step queued on the build seat " + seat + " has waited " + CapacityBound.String() +
|
|
" for a build node to take it.",
|
|
Needs: "build capacity: a build node present and not paused, or fewer steps asked"}
|
|
left := CapacityBound - w.now().Sub(oldest)
|
|
if left > 0 {
|
|
w.unfind(key)
|
|
w.signal(capacityRow, seat, left, signalDue{Do: dueFind, Finding: f})
|
|
return
|
|
}
|
|
w.find(f)
|
|
}
|