Files
mesh-catalog/modules/mesh-delivery/cmd/mesh-delivery/model.go
T
jochen 404efb764b
mesh/merge-gate fail: builds mesh-delivery → novox; no bus step; a manifest the change touches fails the module check: mesh-delivery claims mesh-delivery a…
mesh/repo-check fail: its merge-check.sh failed: files asking for a setting without saying whether it is trusted: 25
mesh/delivery superseded: a newer head of the same pull request
mesh-delivery: keep each delivery's time from its merge and answer times per class (hq ADR 0282, issue 382)
The operator's budget (a leaf module running everywhere within five minutes
of its merge, a core module within ten) cannot be held to without measuring
it. Each delivery now keeps its delivery time, class and phases from the
walk the controller says; 'times' answers the week's median and worst per
class, the deliveries over budget and their longest phase, and one
delivery's breakdown; 'deliveries' is headed by the same line. A phase the
walk could not measure stays unknown, never a zero. Measurement only.
2026-10-10 20:14:19 +02:00

637 lines
24 KiB
Go

package main
import (
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
"time"
)
// What mesh-delivery owns (novox/hq ADR 0239): the delivery — one commit in one repository — and the
// delivery group above it. Each is kept whole, one key of the module's own state per delivery and per
// group, written only by this holder.
// Delivery is one commit in one repository, from its pull request's head to every machine.
type Delivery struct {
// ID is owner/repository@<twelve characters of its commit>.
ID string `json:"id"`
Repository string `json:"repository"`
Commit string `json:"commit"`
// The pull request, when there is one: its number, title, base, head branch and page.
Number int `json:"number,omitempty"`
Title string `json:"title,omitempty"`
Base string `json:"base,omitempty"`
Branch string `json:"branch,omitempty"`
HTMLURL string `json:"html_url,omitempty"`
CloneURL string `json:"clone_url,omitempty"`
// The diffset, as the forge said it: what the planner maps onto modules.
Paths []string `json:"paths,omitempty"`
PathsTruncated bool `json:"paths_truncated,omitempty"`
Removed []string `json:"removed,omitempty"`
ModuleDirs []string `json:"module_dirs,omitempty"`
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
// After are the repositories its pull request says it goes after (`after: <repository>` lines).
After []string `json:"after,omitempty"`
// Group is the delivery group it is in: its head branch name. Empty for a lone delivery.
Group string `json:"group,omitempty"`
State State `json:"state"`
Since time.Time `json:"since"`
Created time.Time `json:"created"`
// Plan is its delivery plan, as the controller's planner computed it (the build plan, the deploy plan,
// the steps that are not an ordinary send).
Plan *DeliveryPlan `json:"plan,omitempty"`
// Check is its own check's verdict.
Check *Verdict `json:"check,omitempty"`
// CheckAsk is this owner's own ask of its check, when the forge's announcement did not bring one: a head
// adopted while it already waited, or one whose announcement the controller missed.
CheckAsk *CheckAsk `json:"check_ask,omitempty"`
// The facts the table's guards read. NewerHead is the head that superseded it; ClosedUnmerged the pull
// request closed; MergedAs the commit it landed on the trunk as; HeldWhy what holds it for a person.
NewerHead string `json:"newer_head,omitempty"`
ClosedUnmerged bool `json:"closed_unmerged,omitempty"`
MergedAs string `json:"merged_as,omitempty"`
MergedAt time.Time `json:"merged_at"`
HeldWhy string `json:"held_why,omitempty"`
// FromForge is a delivery made from the forge's word at its merge: its head was never heard by this owner.
// History is why it is history, once it is (novox/hq issue 313).
FromForge bool `json:"from_forge,omitempty"`
History string `json:"history,omitempty"`
// CoveredBy says which later merges into the same trunk delivered every module it moves, once they did
// (novox/hq issue 319); Lifted why what held it for a person no longer holds (issue 316).
CoveredBy string `json:"covered_by,omitempty"`
Lifted string `json:"lifted,omitempty"`
// ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's
// merge (gitea_contains): asked once per pair, and kept (ADR 0270).
ContainedIn map[string]bool `json:"contained_in,omitempty"`
// AnsweredBy is the commit of its repository the walk it follows carries for it: its own merge, or a later merge
// of its branch that contains it (novox/hq ADR 0276). Unwalked says it is held because no walk answered it.
AnsweredBy string `json:"answered_by,omitempty"`
// Carrier says, for a merge a later one carries, which: its delivery and walk, in words.
Carrier string `json:"carrier,omitempty"`
Unwalked bool `json:"unwalked,omitempty"`
// Walk is the walk that delivers it, as last read: its own trunk commit's, or one that carries a later merge
// of its branch; Steps the machines it reached.
Walk *WalkSeen `json:"walk,omitempty"`
Steps []Step `json:"steps,omitempty"`
// Time is its delivery time, class and phases (novox/hq ADR 0282), once its walk's end is known.
Time *DeliveryTime `json:"delivery_time,omitempty"`
// LetGo is when this owner asked the controller to start its walk, and why.
LetGo time.Time `json:"let_go"`
LetGoWhy string `json:"let_go_why,omitempty"`
Transitions []Transition `json:"transitions,omitempty"`
// Owed are the forge's and the bus's effects of its transitions not yet done: said only after the
// delivery is kept, retried until done.
Owed []Effect `json:"owed,omitempty"`
}
// Transition is one move of the table, kept.
type Transition struct {
At time.Time `json:"at"`
From State `json:"from"`
To State `json:"to"`
Event Event `json:"event"`
Why string `json:"why"`
By string `json:"by,omitempty"`
}
// Verdict is a check's verdict: the gate's and the repository's own. ID is the build seat's ask that ran
// it, On the machine it ran on, At when this owner heard it; Report is the check's own account, as bounded
// as the controller says it. All of it is what `checks` answers, whole, beside the forge's short statuses.
type Verdict struct {
ID string `json:"id,omitempty"`
On string `json:"on,omitempty"`
At time.Time `json:"at"`
Gate string `json:"gate"`
Summary string `json:"summary"`
Modules []string `json:"modules,omitempty"`
Dependents []string `json:"dependents,omitempty"`
Repo string `json:"repo,omitempty"`
RepoSaid string `json:"repo_summary,omitempty"`
Report string `json:"report,omitempty"`
}
// CheckAsk is a head's own check, asked of the controller by this owner (or re-asked by healer H2).
type CheckAsk struct {
// ID is the ask's id as the controller answered it.
ID string `json:"id"`
At time.Time `json:"at"`
By string `json:"by"`
Why string `json:"why"`
// Reasked is how many times H2 asked it again: once at most, then the operator's.
Reasked int `json:"reasked,omitempty"`
}
// Passes is whether the verdict lets the delivery be ready: the gate passed or warned, and the
// repository's own check did not fail or error.
func (v *Verdict) Passes() bool {
if v == nil {
return false
}
gate := v.Gate == "pass" || v.Gate == "warning"
repo := v.Repo == "" || v.Repo == "pass" || v.Repo == "warning"
return gate && repo
}
// DeliveryPlan is a delivery plan as the controller's planner says it (mesh-controller link.ChangePlan).
type DeliveryPlan struct {
Repository string `json:"repository"`
Base string `json:"base"`
Head string `json:"head"`
Moved []string `json:"moved,omitempty"`
Dependents []string `json:"dependents,omitempty"`
New []string `json:"new,omitempty"`
Unread []string `json:"unread,omitempty"`
Tiers [][]string `json:"tiers,omitempty"`
Machines []MachinePlan `json:"machines,omitempty"`
Steps []string `json:"steps,omitempty"`
Summary string `json:"summary"`
}
// receiving are the machines the plan sends a module to, in the plan's order; none when the plan names no
// machines, as a plan kept before they were said may not.
func (p *DeliveryPlan) receiving(module string) []string {
if p == nil {
return nil
}
var out []string
for _, m := range p.Machines {
if containsString(m.Receives, module) {
out = append(out, m.Machine)
}
}
return out
}
// MachinePlan is one machine's part of a plan.
type MachinePlan struct {
Machine string `json:"machine"`
Receives []string `json:"receives,omitempty"`
Waits []string `json:"waits,omitempty"`
}
// waitsForItsOwnWalk is whether it is published again after the walk that carried a later commit for it failed.
func (d *Delivery) waitsForItsOwnWalk() bool {
return d.State == Published && d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater()
}
// carriedLater is whether the walk it follows carries a later merge of its branch for it, not its own.
func (d *Delivery) carriedLater() bool {
return d.AnsweredBy != "" && d.MergedAs != "" && !sameCommit(d.AnsweredBy, d.MergedAs)
}
// MovesNothing is whether the delivery's plan moves no module the mesh holds: nothing for a walk to do. A
// module it adds is not moved: the controller builds and registers it and sends it nowhere, and opens no walk
// for it (novox/hq issues 300 and 319). A delivery with no plan known is not said to move nothing.
func (d *Delivery) MovesNothing() bool {
return d.Plan != nil && len(d.Plan.Moved) == 0
}
// Step is one machine's part of a delivering walk.
type Step struct {
Machine string `json:"machine"`
Module string `json:"module"`
Build string `json:"build,omitempty"`
State StepState `json:"state"`
Since time.Time `json:"since"`
Why string `json:"why,omitempty"`
}
// Effect is one thing owed outside the state: an event said, a line appended to a commit's note, the view
// on the pull request refreshed, a status set.
type Effect struct {
Kind string `json:"kind"`
Commit string `json:"commit,omitempty"`
Line string `json:"line,omitempty"`
Context string `json:"context,omitempty"`
State string `json:"state,omitempty"`
Tries int `json:"tries,omitempty"`
Last string `json:"last,omitempty"`
Since time.Time `json:"since"`
// Next is when a failed effect is tried again: later after every failure, up to retryCap.
Next *time.Time `json:"next,omitempty"`
}
// The kinds of effect.
const (
EffectEmit = "emit"
EffectNote = "note"
EffectView = "view"
EffectStatus = "status"
)
// FromForgesWord is whether the delivery was made from the forge's word at its merge: said by FromForge, or —
// for one made before FromForge was kept — by its first transition.
func (d *Delivery) FromForgesWord() bool {
if d.FromForge {
return true
}
return len(d.Transitions) > 0 && d.Transitions[0].Event == EvAnnounced &&
strings.Contains(d.Transitions[0].Why, "so it is made from the forge's word")
}
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
// pull request the forge may no longer hold — or holds under its number another one.
func (d *Delivery) shownOnItsPullRequest() bool {
return d.Number > 0 && d.History == "" && d.State != Retired
}
// Owner and Repo split the repository.
func (d *Delivery) Owner() string { o, _, _ := strings.Cut(d.Repository, "/"); return o }
func (d *Delivery) Repo() string { _, r, _ := strings.Cut(d.Repository, "/"); return r }
// IDOf is a delivery's id: the repository and twelve characters of its commit.
func IDOf(repository, commit string) string {
return strings.ToLower(repository) + "@" + shortOf(commit, 12)
}
func shortOf(commit string, n int) string {
if len(commit) > n {
return commit[:n]
}
return commit
}
// sameCommit is whether two commits are one, however either is abbreviated.
func sameCommit(a, b string) bool {
if a == "" || b == "" {
return false
}
if len(a) > len(b) {
a, b = b, a
}
return len(a) >= 7 && strings.HasPrefix(strings.ToLower(b), strings.ToLower(a))
}
// afterLines are the `after: <repository>` lines of a pull request's description.
var afterLines = regexp.MustCompile(`(?im)^\s*after:\s*([A-Za-z0-9_.\-/]+)\s*$`)
// AfterIn reads them.
func AfterIn(body string) []string {
var out []string
for _, m := range afterLines.FindAllStringSubmatch(body, -1) {
out = append(out, strings.TrimSuffix(m[1], ".git"))
}
sort.Strings(out)
return out
}
// The walk: the controller's plan of one trunk commit, as its `plan-moved` and `delivery-walks` say it
// (mesh-controller inventory.Plan). Read, never written here.
// The states of a walk as the controller keeps them.
const (
walkBuilding = "building"
walkRolling = "rolling"
walkDone = "done"
walkFailed = "failed"
walkSuperseded = "superseded"
)
// Walk is a walk whole, as the controller says it.
type Walk struct {
ID string `json:"id"`
Repository string `json:"repository"`
Branch string `json:"branch,omitempty"`
Commit string `json:"commit"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
State string `json:"state"`
Tier int `json:"tier"`
Tiers [][]string `json:"tiers"`
Modules map[string]*WalkModule `json:"modules"`
Note string `json:"note,omitempty"`
Revision int64 `json:"revision"`
Release *WalkRelease `json:"release,omitempty"`
Delivery *WalkDelivery `json:"delivery,omitempty"`
// Phases is the walk's time from its merge, phase by phase, as the controller works it out (novox/hq ADR
// 0282): what a delivery's time is kept from. Absent from a controller that does not say it.
Phases *WalkPhases `json:"phases,omitempty"`
}
// WalkModule is one module of a walk.
type WalkModule struct {
State string `json:"state,omitempty"`
SentAt *time.Time `json:"sent_at,omitempty"`
First []string `json:"first,omitempty"`
FirstAt *time.Time `json:"first_at,omitempty"`
Commit string `json:"commit,omitempty"`
Why string `json:"why,omitempty"`
Build string `json:"build,omitempty"`
Gate *WalkGate `json:"gate,omitempty"`
GatedBy string `json:"gated_by,omitempty"`
}
// WalkGate is a module's judging on its first machine.
type WalkGate struct {
Machines []string `json:"machines"`
From string `json:"from,omitempty"`
To string `json:"to,omitempty"`
Since *time.Time `json:"since,omitempty"`
Passes int `json:"passes,omitempty"`
Verdict string `json:"verdict,omitempty"`
Why string `json:"why,omitempty"`
JudgedAt *time.Time `json:"judged_at,omitempty"`
Rollback string `json:"rollback,omitempty"`
Carried []WalkCarry `json:"carried,omitempty"`
}
// WalkCarry is one build a gated send carried.
type WalkCarry struct {
Module string `json:"module"`
Node string `json:"node"`
From string `json:"from,omitempty"`
To string `json:"to"`
Build string `json:"build,omitempty"`
}
// WalkRelease is a backlog walk's way through the machines.
type WalkRelease struct {
Order []string `json:"order"`
Next int `json:"next"`
Gate *WalkGate `json:"gate,omitempty"`
Done []string `json:"done,omitempty"`
By string `json:"by,omitempty"`
}
// WalkDelivery is what a walk waits for, and which merges it answers (novox/hq ADR 0276).
type WalkDelivery struct {
Awaits string `json:"awaits"`
Go *time.Time `json:"go,omitempty"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
Stopped string `json:"stopped,omitempty"`
StoppedWhy string `json:"stopped_why,omitempty"`
// Merges are every merge the walk answers: each merge commit, its repository, and the commit of that
// repository the walk carries — the merge itself, or a later merge of its branch that contains it. Absent
// from a controller that does not say it: the walk then answers the merge of its own commit, and a merge
// the forge says its commit contains.
Merges []WalkMerge `json:"merges,omitempty"`
// TakenOverBy is, on a walk folded before it started, the walk that took it over and answers every merge
// it answered.
TakenOverBy string `json:"taken_over_by,omitempty"`
// Batch is set while the record is a batch, not yet a walk: its state `assembling` or `queued`. A batch, once
// cut, is the walk under the same id: its next record is `building`. A merge this owner reads in a batch is
// not held as unwalked while the batch is heard of.
Batch *WalkBatch `json:"batch,omitempty"`
}
// WalkMerge is one merge a walk answers.
type WalkMerge struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Carried is the commit of the repository the walk carries for it: Commit itself, or a later merge of its
// branch that contains it. Empty means Commit.
Carried string `json:"carried,omitempty"`
}
// WalkBatch is a batch's window: when it closes unless another merge arrives, when it closes at the latest,
// and, once closed, the walk it waits behind.
type WalkBatch struct {
ClosesAt *time.Time `json:"closes_at,omitempty"`
AtMost *time.Time `json:"at_most,omitempty"`
Behind string `json:"behind,omitempty"`
}
// The states a batch's record is in before it is a walk (novox/hq ADR 0276).
const (
walkAssembling = "assembling"
walkQueued = "queued"
)
// IsBatch is whether the record is a batch still to be cut into a walk.
func (w Walk) IsBatch() bool { return w.State == walkAssembling || w.State == walkQueued }
// supersededNote reads the walk that took a superseded walk over from its note, for a controller that does not
// say TakenOverBy: "superseded at tier N by plan-… (…)".
var supersededNote = regexp.MustCompile(`^superseded at tier \d+ by (plan-[0-9A-Za-z_-]+)`)
// takenOverBy is the walk that took this one over, said or read from its note; empty for none.
func (w Walk) takenOverBy() string {
if w.State != walkSuperseded {
return ""
}
if w.Delivery != nil && w.Delivery.TakenOverBy != "" {
return w.Delivery.TakenOverBy
}
if m := supersededNote.FindStringSubmatch(w.Note); m != nil {
return m[1]
}
return ""
}
// carriedFor is the commit the walk carries for a merge of a repository, and whether it answers that merge by
// what it says: the merge named among its merges, or its own commit.
func (w Walk) carriedFor(repository, merge string) (string, bool) {
if w.Delivery != nil {
for _, m := range w.Delivery.Merges {
if strings.EqualFold(m.Repository, repository) && sameCommit(m.Commit, merge) {
if m.Carried != "" {
return m.Carried, true
}
return m.Commit, true
}
}
}
if strings.EqualFold(w.Repository, repository) && sameCommit(w.Commit, merge) {
return w.Commit, true
}
return "", false
}
// WalkSeen is what a delivery keeps of its walk.
type WalkSeen struct {
ID string `json:"id"`
Commit string `json:"commit"`
State string `json:"state"`
Note string `json:"note,omitempty"`
Revision int64 `json:"revision"`
Waits bool `json:"waits,omitempty"`
WaitedFor string `json:"waited_for,omitempty"`
GoBy string `json:"go_by,omitempty"`
StoppedBy string `json:"stopped_by,omitempty"`
Tier int `json:"tier"`
Tiers int `json:"tiers"`
Updated time.Time `json:"updated"`
// Branch is the branch the walk's commit was merged into; empty in a record kept before it was read.
Branch string `json:"branch,omitempty"`
// Created is when the controller made the walk: a delivery follows the newest walk that answers it.
Created time.Time `json:"created"`
// TakenOverBy is the walk that took a superseded one over (novox/hq ADR 0276): its deliveries follow it.
TakenOverBy string `json:"taken_over_by,omitempty"`
}
// Seen is a walk as a delivery keeps it.
func (w Walk) Seen() *WalkSeen {
s := &WalkSeen{ID: w.ID, Commit: w.Commit, State: w.State, Note: w.Note, Revision: w.Revision, Tier: w.Tier,
Tiers: len(w.Tiers), Updated: w.Updated}
if d := w.Delivery; d != nil {
s.WaitedFor = d.Awaits
s.Waits = d.Awaits != "" && d.Go == nil
s.GoBy = d.By
s.StoppedBy = d.Stopped
}
s.Branch = w.Branch
s.Created = w.Created
s.TakenOverBy = w.takenOverBy()
return s
}
// Started is whether the walk is past its wait: it waited for nobody, or was let go — whatever came of it
// since. A walk superseded or stopped while it waited never started.
func (s *WalkSeen) Started() bool { return s != nil && !s.Waits && (s.WaitedFor == "" || s.GoBy != "") }
// Waited is whether it waited for a word at all.
func (s *WalkSeen) Waited() bool { return s != nil && s.WaitedFor != "" }
// LetGoByAPerson is whether a person's `plans go` started it.
func (s *WalkSeen) LetGoByAPerson() bool { return s != nil && strings.HasPrefix(s.GoBy, "a person") }
// Open is whether the controller still works it.
func (w Walk) Open() bool { return w.State == walkBuilding || w.State == walkRolling }
// The steps that say a module reached every machine running it: sent to all at once, or to the rest after its
// first machines passed.
const (
stepEveryMachine = "every machine running it"
stepTheRest = "the rest running it"
)
// StepsOf are the machine steps a walk's record says, for one delivery: every module it moved, on its first
// machines from its gate, and the rest once sent.
func StepsOf(w Walk) []Step {
var out []Step
names := make([]string, 0, len(w.Modules))
for n := range w.Modules {
names = append(names, n)
}
sort.Strings(names)
for _, name := range names {
m := w.Modules[name]
if m == nil || m.FirstAt == nil {
if m != nil && m.SentAt != nil && m.FirstAt == nil && m.Why == "" {
out = append(out, Step{Machine: stepEveryMachine, Module: name, Build: m.Build,
State: StepSent, Since: *m.SentAt})
}
continue
}
gate := m.Gate
state, why, since := StepSent, "", *m.FirstAt
if gate != nil {
switch {
case gate.Rollback == "rolled-back":
state, why = StepRolledBack, gate.Why
case gate.Verdict == "failed":
state, why = StepFailed, gate.Why
case gate.Verdict == "passed":
state, why = StepPassed, gate.Why
case gate.Since != nil:
state = StepJudging
why = fmt.Sprintf("%d healthy judging(s) so far", gate.Passes)
}
if gate.JudgedAt != nil {
since = *gate.JudgedAt
}
}
for _, machine := range m.First {
out = append(out, Step{Machine: machine, Module: name, Build: m.Build, State: state, Since: since, Why: why})
}
if m.SentAt != nil && state == StepPassed {
out = append(out, Step{Machine: stepTheRest, Module: name, Build: m.Build, State: StepSent,
Since: *m.SentAt})
}
}
return out
}
// Group is a delivery group: two or more deliveries sharing a head branch name, one level only.
type Group struct {
ID string `json:"id"`
Members []string `json:"members"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
// Order is the members in the order they are delivered, Pairs why, Cycle the members whose order
// contradicts itself; OrderOf the member commits the order was worked out for.
Order []string `json:"order,omitempty"`
Pairs []Pair `json:"pairs,omitempty"`
Cycle []string `json:"cycle,omitempty"`
// Contradictions is each two members whose rules order them both ways at one rank, said with how a
// person declares the order (novox/hq ADR 0249).
Contradictions []Contradiction `json:"contradictions,omitempty"`
OrderOf map[string]string `json:"order_of,omitempty"`
// Check is its composed check: asked of the controller for these member commits, and its verdict.
Check *GroupCheck `json:"check,omitempty"`
// Closed is set once a member reached the trunk: a group delivering takes no new member.
Closed bool `json:"closed,omitempty"`
// Said is the derived state last said, so a change of it is said once.
Said string `json:"said,omitempty"`
Owed []Effect `json:"owed,omitempty"`
}
// Pair is one "before" among a group's members: the rule that won, and the rules that ordered it the other
// way and lost by precedence (novox/hq ADR 0249).
type Pair struct {
Before string `json:"before"`
After string `json:"after"`
Why string `json:"why"`
Over []string `json:"over,omitempty"`
}
// Contradiction is two members that rules of one rank order both ways, which no precedence resolves: the
// rules, and Said, which names them and says how to declare the order.
type Contradiction struct {
Members []string `json:"members"`
Rules []Pair `json:"rules"`
Said string `json:"said"`
}
// orderSaid is a pair as a person reads it: what won, and over what.
func (p Pair) orderSaid() string {
s := p.Before + " before " + p.After + ": " + p.Why
if len(p.Over) > 0 {
s += ", over " + strings.Join(p.Over, ", ")
}
return s
}
// GroupCheck is a group's composed check.
type GroupCheck struct {
Asked string `json:"asked"`
At time.Time `json:"at"`
Commits map[string]string `json:"commits"`
Verdict string `json:"verdict,omitempty"`
Summary string `json:"summary,omitempty"`
}
// Commits is each member's commit, by id.
func commitsOf(ds []*Delivery) map[string]string {
out := map[string]string{}
for _, d := range ds {
out[d.ID] = d.Commit
}
return out
}
// sameMembers is whether two member→commit maps are one.
func sameMembers(a, b map[string]string) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
if b[k] != v {
return false
}
}
return true
}
func mustJSON(v any) json.RawMessage {
raw, _ := json.Marshal(v)
return raw
}