Files
mesh-controller/internal/inventory/plans.go
T
jochen bb1607e424 Say when the mesh is wrong: conditions, watchdogs, the bus's advisories, doctor (hq to-be 45 Phase 1)
Every one of the 48 core failures of research 031 was found by a person
looking; the mesh's answers carried the fact for whoever asked and told
nobody.

- The condition store (to-be 45 §2): mesh-controller_conditions, one key
  per open condition, written by compare-and-set so a person's silence
  and the watchdogs never lose each other's word; every transition kept
  ninety days in mesh-controller_condition-history and said as the
  seat's events condition-raised / condition-changed / condition-cleared
  (the condition at the top level, with event, at, change, why, show),
  offered again while the bus is away. Raised and cleared by observation
  only; a clearing reopened within ten minutes is the same condition with
  its count up, its silence kept. Verbs: conditions, conditions show,
  conditions silence (a hand act, at most a week), conditions history.
- ADR 0224's provider standing is the first kind, provider-failing, held
  by the provider's events; the provider_standing table is no longer read
  or written (left in place: dropping it is the operator's word).
- status leads with the open conditions, urgent first, and says all well
  only with none open; conditions it cannot read are said and not well.
- The signals table compiled in, one watchdog loop over it every 30s: S1
  heartbeat (3 intervals, asleep machines excepted, control node urgent
  after 30 min), S2 report after a send, S3 plan tier, S4 event loop deaf,
  S5 merge not acted, S6 ask lost, S7 call hung, S8 provider silent, S9
  advisories, S10 self-check silent, S11 node tools silent, S13 stale
  refusals; S12, S14, S15 deferred with their reasons. A row that cannot
  see raises probe-failed and clears nothing. A test generated from the
  table suppresses each signal inside and past its bound.
- The bus's advisories (maximum deliveries, a mesh consumer deleted) and
  the controller's own slow consumer and refused subjects, said in the
  mesh's words.
- doctor: the probe registry D1-D10 (D5 deferred) and DW, every five
  minutes, each in thirty seconds; a probe that cannot run is never a
  pass. D1 validates with mesh-host's own validator. Every run ends with
  the doctor-heartbeat event mesh-watcher listens for.
- The controller is granted its new buckets, events, the two advisories
  and $SRV.INFO; the node tools their tools-alive heartbeat. The streams
  and consumers the controller asserts and the ones D6/D7 expect are one
  derivation.
2026-10-06 10:21:11 +02:00

167 lines
6.4 KiB
Go

package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
// still waits for.
type Plan struct {
ID string `json:"id"`
Repository string `json:"repository"`
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
// ones of the same repository and branch. Empty for a plan from before it was kept.
Branch string `json:"branch,omitempty"`
Commit string `json:"commit"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
State string `json:"state"`
Tier int `json:"tier"`
Tiers [][]string `json:"tiers"`
Modules map[string]*PlanModule `json:"modules"`
Note string `json:"note,omitempty"`
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
// never written from here: a save measures the tier it leaves and stamps the next. What the
// watchdog of a plan's progress (S3) reads.
TierEntered time.Time `json:"tier_entered,omitempty"`
}
// PlanModule is one module's state within a plan.
type PlanModule struct {
// State: asked, built, failed; empty for a module whose tier has not been asked yet.
State string `json:"state,omitempty"`
AskedAt *time.Time `json:"asked_at,omitempty"`
BuiltAt *time.Time `json:"built_at,omitempty"`
// SentAt is when the plan sent the machines running this module its new build, because a
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
// after this.
SentAt *time.Time `json:"sent_at,omitempty"`
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
// replaced while the plan waits on that report resumes the wait rather than sending again. The
// machine holding the bus is among them when its user list had to go first.
First []string `json:"first,omitempty"`
FirstAt *time.Time `json:"first_at,omitempty"`
Commit string `json:"commit,omitempty"`
Why string `json:"why,omitempty"`
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
// its module's name included. Empty in a plan from before it was kept, which is matched by
// module, or by repository and path, as before.
Build string `json:"build,omitempty"`
}
// The states a plan passes through.
const (
PlanBuilding = "building"
PlanRolling = "rolling"
PlanDone = "done"
PlanFailed = "failed"
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
PlanSuperseded = "superseded"
)
// Open says whether the plan is still being worked.
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
tiers, err := json.Marshal(p.Tiers)
if err != nil {
return err
}
modules, err := json.Marshal(p.Modules)
if err != nil {
return err
}
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
// measure one twice.
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
entered, err := planTierLeft(ctx, tx, p, time.Now())
if err != nil {
return err
}
_, err = tx.Exec(ctx,
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
tier_entered = excluded.tier_entered`,
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
if err != nil {
return err
}
return tx.Commit(ctx)
}
// OpenPlans is every plan still being worked, oldest first.
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
}
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
}
// PlanByID is one plan.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
plans, err := i.plans(ctx, `where id = '`+id+`'`)
if err != nil {
return Plan{}, err
}
if len(plans) == 0 {
return Plan{}, fmt.Errorf("no plan %s", id)
}
return plans[0], nil
}
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
coalesce(tier_entered, created)
from release_plan `+tail)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Plan
for rows.Next() {
var p Plan
var tiers, modules []byte
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered); err != nil {
return nil, err
}
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
return nil, err
}
if err := json.Unmarshal(modules, &p.Modules); err != nil {
return nil, err
}
if p.Modules == nil {
p.Modules = map[string]*PlanModule{}
}
out = append(out, p)
}
if errors.Is(rows.Err(), pgx.ErrNoRows) {
return nil, nil
}
return out, rows.Err()
}