Every one of the 48 core failures of research 031 was found by a person looking; the mesh's answers carried the fact for whoever asked and told nobody. - The condition store (to-be 45 §2): mesh-controller_conditions, one key per open condition, written by compare-and-set so a person's silence and the watchdogs never lose each other's word; every transition kept ninety days in mesh-controller_condition-history and said as the seat's events condition-raised / condition-changed / condition-cleared (the condition at the top level, with event, at, change, why, show), offered again while the bus is away. Raised and cleared by observation only; a clearing reopened within ten minutes is the same condition with its count up, its silence kept. Verbs: conditions, conditions show, conditions silence (a hand act, at most a week), conditions history. - ADR 0224's provider standing is the first kind, provider-failing, held by the provider's events; the provider_standing table is no longer read or written (left in place: dropping it is the operator's word). - status leads with the open conditions, urgent first, and says all well only with none open; conditions it cannot read are said and not well. - The signals table compiled in, one watchdog loop over it every 30s: S1 heartbeat (3 intervals, asleep machines excepted, control node urgent after 30 min), S2 report after a send, S3 plan tier, S4 event loop deaf, S5 merge not acted, S6 ask lost, S7 call hung, S8 provider silent, S9 advisories, S10 self-check silent, S11 node tools silent, S13 stale refusals; S12, S14, S15 deferred with their reasons. A row that cannot see raises probe-failed and clears nothing. A test generated from the table suppresses each signal inside and past its bound. - The bus's advisories (maximum deliveries, a mesh consumer deleted) and the controller's own slow consumer and refused subjects, said in the mesh's words. - doctor: the probe registry D1-D10 (D5 deferred) and DW, every five minutes, each in thirty seconds; a probe that cannot run is never a pass. D1 validates with mesh-host's own validator. Every run ends with the doctor-heartbeat event mesh-watcher listens for. - The controller is granted its new buckets, events, the two advisories and $SRV.INFO; the node tools their tools-alive heartbeat. The streams and consumers the controller asserts and the ones D6/D7 expect are one derivation.
167 lines
6.4 KiB
Go
167 lines
6.4 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
|
|
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
|
|
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
|
|
// still waits for.
|
|
type Plan struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
|
|
// ones of the same repository and branch. Empty for a plan from before it was kept.
|
|
Branch string `json:"branch,omitempty"`
|
|
Commit string `json:"commit"`
|
|
Created time.Time `json:"created"`
|
|
Updated time.Time `json:"updated"`
|
|
State string `json:"state"`
|
|
Tier int `json:"tier"`
|
|
Tiers [][]string `json:"tiers"`
|
|
Modules map[string]*PlanModule `json:"modules"`
|
|
Note string `json:"note,omitempty"`
|
|
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
|
|
// never written from here: a save measures the tier it leaves and stamps the next. What the
|
|
// watchdog of a plan's progress (S3) reads.
|
|
TierEntered time.Time `json:"tier_entered,omitempty"`
|
|
}
|
|
|
|
// PlanModule is one module's state within a plan.
|
|
type PlanModule struct {
|
|
// State: asked, built, failed; empty for a module whose tier has not been asked yet.
|
|
State string `json:"state,omitempty"`
|
|
AskedAt *time.Time `json:"asked_at,omitempty"`
|
|
BuiltAt *time.Time `json:"built_at,omitempty"`
|
|
// SentAt is when the plan sent the machines running this module its new build, because a
|
|
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
|
|
// after this.
|
|
SentAt *time.Time `json:"sent_at,omitempty"`
|
|
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
|
|
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
|
|
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
|
|
// replaced while the plan waits on that report resumes the wait rather than sending again. The
|
|
// machine holding the bus is among them when its user list had to go first.
|
|
First []string `json:"first,omitempty"`
|
|
FirstAt *time.Time `json:"first_at,omitempty"`
|
|
Commit string `json:"commit,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
|
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
|
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
|
// module, or by repository and path, as before.
|
|
Build string `json:"build,omitempty"`
|
|
}
|
|
|
|
// The states a plan passes through.
|
|
const (
|
|
PlanBuilding = "building"
|
|
PlanRolling = "rolling"
|
|
PlanDone = "done"
|
|
PlanFailed = "failed"
|
|
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
|
|
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
|
|
PlanSuperseded = "superseded"
|
|
)
|
|
|
|
// Open says whether the plan is still being worked.
|
|
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
|
|
|
|
// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
|
|
func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
|
|
tiers, err := json.Marshal(p.Tiers)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
modules, err := json.Marshal(p.Modules)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
|
|
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
|
|
// measure one twice.
|
|
tx, err := i.store.Pool().Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
entered, err := planTierLeft(ctx, tx, p, time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.Exec(ctx,
|
|
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
|
|
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
|
|
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
|
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
|
|
tier_entered = excluded.tier_entered`,
|
|
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit(ctx)
|
|
}
|
|
|
|
// OpenPlans is every plan still being worked, oldest first.
|
|
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
|
|
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
|
|
}
|
|
|
|
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
|
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
|
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
|
|
}
|
|
|
|
// PlanByID is one plan.
|
|
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
|
plans, err := i.plans(ctx, `where id = '`+id+`'`)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
if len(plans) == 0 {
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
return plans[0], nil
|
|
}
|
|
|
|
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
|
|
coalesce(tier_entered, created)
|
|
from release_plan `+tail)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Plan
|
|
for rows.Next() {
|
|
var p Plan
|
|
var tiers, modules []byte
|
|
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
|
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(modules, &p.Modules); err != nil {
|
|
return nil, err
|
|
}
|
|
if p.Modules == nil {
|
|
p.Modules = map[string]*PlanModule{}
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
if errors.Is(rows.Err(), pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
return out, rows.Err()
|
|
}
|