novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

Drop the plan state from the store and the assignments: migration 0108 and placement-only assignments (release 6, issue 1945 B9, B4)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 16:42:02 +02:00
parent 96d7784bce
commit 788fb681ec
110 changed files with 512 additions and 10584 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/chain/chain_test.go
+27 -22
View File
@@ -49,23 +49,23 @@ func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testin
func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) {
ctx := context.Background()
c := InMemory(nil)
first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"})
first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"})
if err != nil {
t.Fatal(err)
}
if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) {
t.Fatalf("a create of a key that holds a value answered %v", err)
}
second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first)
second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c2"}, first)
if err != nil {
t.Fatal(err)
}
// A writer that read the first revision writes now.
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) {
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "stale"}, first); !errors.Is(err, ErrMoved) {
t.Fatalf("a write at a moved revision answered %v", err)
}
got, _, _ := c.Assignments.Get(ctx, "anchor.notes")
if got.Value.Commit != "c2" || got.Revision != second {
if got.Value.RequiredBy != "c2" || got.Revision != second {
t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision)
}
}
@@ -91,8 +91,8 @@ func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) {
}
ctx := context.Background()
c := InMemory(nil)
rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"})
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil {
rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"})
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}, rev); err != nil {
t.Fatal(err)
}
keys, err := c.Assignments.Keys(ctx)
@@ -190,31 +190,36 @@ func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing
}
}
// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read
// under the names they read them by (ADR 0408 §3).
func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) {
// **An assignment is placement only** (ADR 0539 §12): the node, the module, since when, what requires it and its
// provisioning plan; no build, artifact, package, commit, rollout or time since is written in it, and a value of before
// release 6 that names them is read with them ignored.
func TestAnAssignmentIsPlacementOnly(t *testing.T) {
a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki",
Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)}
Provisioning: json.RawMessage(`{"plan":1}`)}
raw, _ := json.Marshal(a)
for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`,
`"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`,
`"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} {
`"required-by":"wiki"`, `"provisioning":{"plan":1}`} {
if !strings.Contains(string(raw), want) {
t.Errorf("the assignment's value lacks %s: %s", want, raw)
}
}
for _, not := range []string{`"build"`, `"artifact"`, `"package"`, `"commit"`, `"rollout"`, `"since"`, `"pulled-in"`, `"sent-unknown"`} {
if strings.Contains(string(raw), not) {
t.Errorf("the assignment's value names %s: %s", not, raw)
}
}
if a.Key() != "anchor.notes" {
t.Errorf("key %s", a.Key())
}
// An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did.
raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)})
if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` {
t.Errorf("an assignment without a chain is %s", raw)
// A value written before release 6 is read, its chain fields ignored, and written back without them.
var old Assignment
if err := json.Unmarshal([]byte(`{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z","build":"b2","commit":"c2",`+
`"rollout":"r1","since":"2026-10-18T04:00:00Z","pulled-in":true}`), &old); err != nil {
t.Fatal(err)
}
// A package in place of an artifact.
raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}})
if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) {
t.Errorf("the package is %s", raw)
raw, _ = json.Marshal(old)
if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` {
t.Errorf("an old value written again is %s", raw)
}
}
@@ -341,7 +346,7 @@ func TestAnEmptyBucketIsReadAsEmpty(t *testing.T) {
func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) {
ctx := context.Background()
c := InMemory(nil)
rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), Commit: "c1"})
rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), RequiredBy: "c1"})
if err != nil {
t.Fatal(err)
}
@@ -355,7 +360,7 @@ func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) {
t.Fatal(err)
}
got, found, _ := c.Assignments.Get(ctx, "anchor.notes")
if !found || got.Value.Stands() || got.Value.Commit != "c1" || got.Value.Ended.IsZero() {
if !found || got.Value.Stands() || got.Value.RequiredBy != "c1" || got.Value.Ended.IsZero() {
t.Fatalf("the ended value is %+v (found %v)", got.Value, found)
}
if history, _ := c.Assignments.History(ctx, "anchor.notes"); len(history) != 2 || !history[0].Value.Stands() {
internal/chain/pending_test.go
-24
View File
@@ -61,27 +61,3 @@ func TestTheTransitionsOfAPendingAssignment(t *testing.T) {
t.Error("a claim moves a wait to applying and back")
}
}
// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit
// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have
// none; and unknown is never written in `commit`.
func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) {
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at}
if !pulled.Stands() || pulled.State != "" {
t.Fatalf("a pulled-in value does not stand: %+v", pulled)
}
raw, _ := json.Marshal(pulled)
if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) {
t.Fatalf("the pulled-in value is %s", raw)
}
unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at}
raw, _ = json.Marshal(unknown)
if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() {
t.Fatalf("the hand-sent value is %s", raw)
}
plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at})
if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") {
t.Fatalf("a value that is neither says so: %s", plain)
}
}
internal/chain/records.go
+4 -18
View File
@@ -175,9 +175,10 @@ type Rollout struct {
Links
}
// Assignment is one module on one node, kept under `<node>.<module>`. The first five fields are the ones its readers
// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the
// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when.
// Assignment is one module on one node, kept under `<node>.<module>`: placement only (novox/hq ADR 0539 §12). The first
// five fields are the ones its readers have always read, with the names they have always had (novox/hq ADR 0408 §3);
// no build, commit or rollout is written in it any more, and a value written before release 6 that names them is read
// with them ignored.
type Assignment struct {
Node string `json:"node"`
Module string `json:"module"`
@@ -185,13 +186,6 @@ type Assignment struct {
RequiredBy string `json:"required-by,omitempty"`
Provisioning json.RawMessage `json:"provisioning,omitempty"`
Build string `json:"build,omitempty"`
Artifact string `json:"artifact,omitempty"`
Package *Package `json:"package,omitempty"`
Commit string `json:"commit,omitempty"`
Rollout string `json:"rollout,omitempty"`
Since time.Time `json:"since,omitzero"`
// Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed**
// (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a
// State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that
@@ -201,14 +195,6 @@ type Assignment struct {
// Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when
// it is made, carried into the value that stands: the record of what became of the wait.
Pending *Pending `json:"pending,omitempty"`
// PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a
// send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read
// (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine.
PulledIn bool `json:"pulled-in,omitempty"`
// SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the
// commit is cleared with it. Absent normally; a send that names its builds clears it.
SentUnknown time.Time `json:"sent-unknown,omitzero"`
}
// StatePending is the State of a value that holds no assignment but a pending one.
internal/inventory/batches.go
-138
View File
@@ -1,138 +0,0 @@
package inventory
import (
"context"
"encoding/json"
"strings"
"time"
)
// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a
// module follows is kept here once, with the batch or walk that answers it. A merge is never dropped and never
// walked twice: its key is its repository and commit, and it belongs to one plan at a time.
// BatchedMerge is one merge as the controller heard it.
type BatchedMerge struct {
Repository string
Branch string
Commit string
// Merged is when the forge made it: the branch's order. Heard is when the controller heard it: what the
// window is measured from.
Merged time.Time
Heard time.Time
// Event is the forge's announcement, what the cut plans from.
Event json.RawMessage
// Plan is the batch or walk answering it; empty while it waits to be walked alone.
Plan string
// Alone says it is walked on its own commit, after a failed walk that carried it in a later one.
Alone bool
}
// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan.
func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) {
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
return i.chainAddMerge(ctx, c, m)
}
// MergeOf is one kept merge; false when it was never heard.
func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) {
c, err := i.theChain(ctx)
if err != nil {
return BatchedMerge{}, false, err
}
return i.chainMergeOf(ctx, c, repository, commit)
}
// MergesOf is every merge a plan answers, oldest merge first.
func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainMergesOf(ctx, c, plan)
}
// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the
// newest first.
func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
}
// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a
// module's last look says (novox/hq ADR 0276).
func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) {
c, err := i.theChain(ctx)
if err != nil {
return false, time.Time{}, err
}
return i.chainKeptOnBranch(ctx, c, repository, branch)
}
// EarliestKept says the controller keeps a merge of some repository's branch, and when it heard the first of all: where its record of
// merges begins on the whole mesh (novox/hq ADR 0463). A merge made before then is history on every trunk.
func (i *Inventory) EarliestKept(ctx context.Context) (bool, time.Time, error) {
c, err := i.theChain(ctx)
if err != nil {
return false, time.Time{}, err
}
return i.chainEarliestKept(ctx, c)
}
// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3).
func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainAloneMerges(ctx, c)
}
// WaitingMerges is every merge no plan answers that is not walked alone: kept, and waiting for something else to
// happen to it (a first merge into a repository, waiting for mesh-delivery's word on it, issue 1146).
func (i *Inventory) WaitingMerges(ctx context.Context) ([]BatchedMerge, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainWaitingMerges(ctx, c)
}
// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to
// be walked on its own commit.
func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error {
c, err := i.theChain(ctx)
if err != nil {
return err
}
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
}
// HoldMerge puts a merge back to waiting, no plan answering it, with the event it is kept under replaced (novox/hq issue 1173:
// a first merge held for a build-agent that reads its manifest, the hold said in its event).
func (i *Inventory) HoldMerge(ctx context.Context, repository, commit string, event json.RawMessage) error {
c, err := i.theChain(ctx)
if err != nil {
return err
}
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Event: event}})
}
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
// are read so a fault is seen.
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
}
// SameRepository says two spellings of a repository are one.
func SameRepository(a, b string) bool { return strings.EqualFold(a, b) }
internal/inventory/build_lookups.go
+34
View File
@@ -0,0 +1,34 @@
package inventory
import "context"
// Lookups of builds the delivery plans once held, which other code still reads.
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
// recorded (a manifest handed over by hand).
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
c, err := i.theChain(ctx)
if err != nil {
return "", err
}
return i.chainBuildOf(ctx, c, module, commit)
}
// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
// from one that does (the bus's planned step, ADR 0236).
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainBuildFingerprints(ctx, c, module)
}
// short is a commit as a person reads it.
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
internal/inventory/busy_integration_test.go
-66
View File
@@ -1,66 +0,0 @@
//go:build integration
package inventory
import (
"testing"
"time"
)
func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting,
Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}}
keep := func(busy []BusyWait) bool {
t.Helper()
news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1,
Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false })
if err != nil {
t.Fatal(err)
}
return news
}
read := func() []BusyWait {
t.Helper()
reports, err := inv.LastReports(ctx)
if err != nil {
t.Fatal(err)
}
for _, r := range reports {
if r.Node == node {
return r.Busy
}
}
t.Fatal("the machine's report is not read back")
return nil
}
keep(nil)
if got := read(); len(got) != 0 {
t.Fatalf("a report with nothing busy reads back %+v", got)
}
if !keep(waiting) {
t.Error("a container starting to wait is not news")
}
got := read()
if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" {
t.Fatalf("the wait is not read back: %+v", got)
}
again := []BusyWait{waiting[0]}
again[0].Said = "busy: a check, a minute on"
if keep(again) {
t.Error("a wait whose output changed is news")
}
if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle",
Since: since, Until: since.Add(15 * time.Minute)}}) {
t.Error("a container replaced after it waited is not news")
}
if _, ok := WaitingFor(read(), "build-agent"); ok {
t.Error("a container replaced still reads as waiting")
}
}
internal/inventory/busy_test.go
-140
View File
@@ -1,140 +0,0 @@
package inventory
import (
"strings"
"testing"
"time"
)
// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the
// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the
// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said
// the container replaced.
func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) {
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`,
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z",
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1)
w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
sumsUp(t, w)
if w.TotalMS != 282894 {
t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS)
}
busy := phase(w, PhaseBusy, 0)
if busy.State != PhaseMeasured || busy.TookMS != 60000 {
t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy)
}
if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 {
t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged)
}
var names []string
for _, ph := range w.Phases {
names = append(names, ph.Name)
}
if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) {
t.Fatalf("the busy wait is not between the send and the judging: %v", names)
}
}
// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no
// such phase.
func TestAnOpenBusyWaitIsSaid(t *testing.T) {
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3,
"verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`,
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1)
raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1)
p := walkOf(t, raw)
if !p.Modules["mesh-delivery"].Gate.Waiting() {
t.Fatal("a gate whose busy wait has not ended does not say it waits")
}
w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil)
busy := phase(w, PhaseBusy, 0)
if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") {
t.Fatalf("an open busy wait is not said: %+v", busy)
}
if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" {
t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases)
}
}
// A container a report lists waiting is its module's by its id; its output alone is not a new account.
func TestABusyContainerIsItsModules(t *testing.T) {
list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"},
{ID: "web.server", State: "replaced-when-idle"}}
if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" {
t.Fatalf("build-agent's container is not found waiting: %+v", b)
}
if _, ok := WaitingFor(list, "web"); ok {
t.Error("a container replaced reads as waiting")
}
if _, ok := WaitingFor(list, "build"); ok {
t.Error("another module's container is found by a prefix of its name")
}
again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]}
if !sameBusy(list, again) {
t.Error("a wait whose output changed is a new account")
}
if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() {
t.Error("a wait that ended is not a new account")
}
}
// A machine's report of a busy container is kept with its last account and read back beside it, and its
// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store.
// A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept
// out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335).
func TestARestMachineWaitingForABusyContainerIsNeverSilent(t *testing.T) {
c := walk582(t)
shanks := c.walk.Modules["build-agent"].Rest["shanks"]
ref := shanks.Digest + "@" + shanks.At.Format("2006-01-02T15:04:05.000000")
answered := shanks.At.Add(15 * time.Second)
// shanks answered, and said a container waits: its apply row is the answer, marked busy.
c.rows["shanks"][0].Detail = ApplyBusy
at := c.judged.Add(20 * time.Minute) // past ApplySilentAfter from its send
w := c.walk.WalkPhases(at, c.lookup)
var got RestReport
for _, r := range w.Rest {
if r.Node == "shanks" {
got = r
}
}
if got.State != RestBusy || slicesContains(w.Silent, "shanks") {
t.Fatalf("a machine waiting for a busy container reads %q (silent: %v)", got.State, w.Silent)
}
if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waits for a busy container on shanks") {
t.Fatalf("the walk ended, or does not say it waits: %+v", phase(w, PhaseApply, -1))
}
// 17 minutes on, shanks says it replaced the container: its wait is its own duration, and its report is then.
c.rows["shanks"] = append(c.rows["shanks"], Duration{Kind: DurationBusy, Subject: "shanks", Node: "shanks", Ref: ref,
Started: answered, Took: 17 * time.Minute})
w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
sumsUp(t, w)
for _, r := range w.Rest {
if r.Node == "shanks" {
got = r
}
}
if got.State != RestApplied || got.Busy != "17m0s" || !got.Reported.Equal(answered.Add(17*time.Minute)) {
t.Fatalf("a machine replaced after its wait reads %+v", got)
}
if slicesContains(w.Silent, "shanks") || !w.End.Equal(answered.Add(17*time.Minute)) {
t.Fatalf("the walk read shanks as silent, or ended elsewhere: %v %v", w.Silent, w.End)
}
// And the apply time itself is the answer's, never the wait's.
if r, _ := FirstReportOf(c.rows["shanks"], shanks, c.walkSent); !r.Answered.Equal(answered) || r.Busy != 17*time.Minute {
t.Fatalf("the busy wait is not kept apart from the apply: %+v", r)
}
}
func slicesContains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
internal/inventory/chain.go
-8
View File
@@ -25,10 +25,6 @@ var ErrNoChain = errors.New("this kind of record lives on the bus, and this proc
type chainState struct {
mu sync.Mutex
open ChainOpener
// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
// is not saved yet.
view *rolloutView
pending map[string]pendingMerge
// requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out
// under their modules (issue 1121), and laid is that seen.
requireLayout, laid bool
@@ -75,10 +71,6 @@ func (i *Inventory) UseChain(c *chain.Chain) {
i.state.mu.Lock()
defer i.state.mu.Unlock()
i.chain = c
if i.state.view != nil {
i.state.view.stop()
i.state.view = nil
}
}
// UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment
internal/inventory/chain_assignments.go
+6 -16
View File
@@ -124,13 +124,6 @@ func (i *Inventory) isAssigned(ctx context.Context, node, module string) (bool,
return false, nil
}
// raiseGeneration raises the assignment generation, as the store's triggers do for a change of its table (migration
// 0094): in the chain nothing in the store changes with an assignment, so the writer says it.
func (i *Inventory) raiseGeneration(ctx context.Context) error {
_, err := i.store.Pool().Exec(ctx, `update assignment_generation set generation = generation + 1`)
return err
}
// chainAssign puts a module on a machine, and says whether that is new.
func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, module string) (bool, error) {
var known bool
@@ -154,7 +147,7 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node,
switch {
case found && cur.Value.Stands():
// A module only pulled in by another becomes an assignment, keeping what it holds.
cur.Value.Assigned, cur.Value.PulledIn = now, false
cur.Value.Assigned = now
_, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision)
case found:
// One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one
@@ -170,9 +163,6 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node,
if err != nil {
return false, fmt.Errorf("%s is not assigned to %s on the bus: %w", module, node.Name, err)
}
if err := i.raiseGeneration(ctx); err != nil {
return true, err
}
return true, nil
}
return false, fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name)
@@ -195,7 +185,7 @@ func (i *Inventory) chainUnassign(ctx context.Context, c *chain.Chain, node Node
} else if err != nil {
return fmt.Errorf("%s is not taken off %s on the bus: %w", module, node.Name, err)
}
return i.raiseGeneration(ctx)
return nil
}
return fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name)
}
@@ -273,11 +263,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
case keepSkip:
continue
case keepCreate:
next.Assigned, next.PulledIn = time.Time{}, true
next.Assigned = time.Time{}
_, err = c.Assignments.Create(ctx, key, next)
case keepFresh:
// A value that ended, pulled in again: a fresh one over it, the ended one staying in history.
next.Assigned, next.PulledIn = time.Time{}, true
next.Assigned = time.Time{}
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
default:
cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning
@@ -292,8 +282,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
written++
}
for key, cur := range have {
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" {
continue // one a send still carries is on the machine, and ends when a send does not carry it any more
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
continue
}
// A value of a module no longer pulled in, which no assignment names: it ends, and stays.
cur.Value.Ended = time.Now().UTC()
internal/inventory/chain_builds.go
+3 -43
View File
@@ -810,33 +810,11 @@ func fillManifests(ctx context.Context, c *chain.Chain, builds []Build) error {
return nil
}
// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097).
func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) {
rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]bool{}
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
out[id] = true
}
return out, rows.Err()
}
func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) {
all, err := chainBuildsOf(ctx, c, module)
if err != nil {
return nil, nil, err
}
failed, err := i.failedAtAGate(ctx)
if err != nil {
return nil, nil, err
}
ok := worked(all)
byModule(ok)
out := map[string][]sourceRow{}
@@ -853,7 +831,7 @@ func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module
order = append(order, b.Module)
}
out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint,
made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest, failedGate: failed[b.ID]})
made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest})
}
return out, order, nil
}
@@ -1104,8 +1082,7 @@ func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain)
// chainHeldMade is, per module, what the build standing for its newest successful build made. Read from the pointers: the
// module's `worked` (its newest success) and `held` (the first build of the run of one source that it stands on), one record
// of each. A build that failed its gate ends a run (issue 280), and the pointer cannot say where; then the module's builds
// are read.
// of each.
func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[string][]Artifact, error) {
worked, err := i.pointedBuilds(ctx, c, "*.worked")
if err != nil {
@@ -1115,29 +1092,12 @@ func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[stri
if err != nil {
return nil, err
}
failed, err := i.failedAtAGate(ctx)
if err != nil {
return nil, err
}
out := map[string][]Artifact{}
for module, w := range worked {
stand := w
if h, ok := held[module]; ok && !failed[w.ID] {
if h, ok := held[module]; ok {
stand = h
}
if failed[stand.ID] && stand.ID != w.ID {
rows, _, err := i.chainSourceRows(ctx, c, module)
if err != nil {
return nil, err
}
if builds := rows[module]; len(builds) > 0 {
var made []Artifact
if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err == nil {
out[module] = made
}
}
continue
}
out[module] = fromChainBuild(stand).Made
}
return out, nil
internal/inventory/chain_builds_test.go
-1
View File
@@ -123,7 +123,6 @@ func TestTheBusBeingAwayIsNotNowLikeTheStoreBeingAway(t *testing.T) {
for name, err := range map[string]error{
"a value too large": chain.ErrTooLarge,
"a key not taken": chain.ErrBadKey,
"a plan moved": ErrPlanMoved,
"a key that exists": chain.ErrExists,
} {
if Unreachable(err) {
internal/inventory/chain_first_read_lays_out_test.go
+10
View File
@@ -23,3 +23,13 @@ func TestTheFirstReadOfAChainNotLaidOutLaysItOut(t *testing.T) {
t.Fatalf("the read did not lay the builds out: found %v, %v", found, err)
}
}
// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the
// store.
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
t.Helper()
c := chain.InMemory(nil)
inv := &Inventory{state: &chainState{}}
inv.UseChain(c)
return inv, c
}
internal/inventory/chain_links.go
-328
View File
@@ -2,15 +2,7 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -21,44 +13,6 @@ import (
// release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read
// of the rollout lists its assignments.
type rolloutKey struct{}
// RolloutModule is what a rollout sends of one module: the commit and the build of it.
type RolloutModule struct{ Commit, Build string }
type rolloutOf struct {
id string
modules map[string]RolloutModule
}
// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries
// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named
// by the build the mesh holds of its commit and by no rollout.
func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context {
return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules})
}
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, err
}
out := map[string]chain.Ref{}
for _, k := range all {
a := k.Value
if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") {
out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit}
}
}
return out, nil
}
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
c, err := i.theChain(ctx)
@@ -78,285 +32,3 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass
sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module })
return out, nil
}
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh
// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of
// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where
// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried
// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named,
// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs
// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for
// what a machine was sent is read from these keys and from nowhere else.
func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error {
c, err := i.theChain(ctx)
if err != nil {
return fmt.Errorf("what the machine runs cannot be written: %w", err)
}
var node string
if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) {
return nil
} else if err != nil {
return err
}
rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf)
all, err := c.Assignments.All(ctx)
if err != nil {
return fmt.Errorf("the assignments of %s cannot be read: %w", node, err)
}
var failed []error
for _, k := range sortedValues(all) {
a := k.Value
if a.Node != node || !a.Stands() {
continue
}
if builds[a.Module] != "" {
continue // carried, and its commit is written below
}
if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil {
failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err))
}
}
modules := make([]string, 0, len(builds))
for m := range builds {
modules = append(modules, m)
}
sort.Strings(modules)
for _, m := range modules {
if builds[m] == "" {
continue
}
if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil {
failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err))
}
}
return errors.Join(failed...)
}
func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] {
out := make([]chain.Kept[chain.Assignment], 0, len(all))
for key, k := range all {
k.Key = key
out = append(out, k)
}
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
return out
}
// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key
// says what the machine runs is unknown.
func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error {
for try := 0; try < casTries; try++ {
next := k.Value
next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{}
next.SentUnknown = time.Time{}
if hand {
next.SentUnknown = at.UTC()
}
if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout &&
next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) &&
next.SentUnknown.Equal(k.Value.SentUnknown) {
return nil
}
if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) {
cur, found, gerr := c.Assignments.Get(ctx, k.Key)
if gerr != nil {
return gerr
}
if !found || !cur.Value.Stands() {
return nil
}
k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision}
continue
} else {
return err
}
}
return fmt.Errorf("the assignment kept moving under the write")
}
func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error {
key := assignmentKey(node, module)
var build, rolloutID string
if sent, ours := rollout.modules[module]; ours && sent.Commit == commit {
build, rolloutID = sent.Build, rollout.id
}
var published *chain.Published
resolved := false
for try := 0; try < casTries; try++ {
cur, found, err := c.Assignments.Get(ctx, key)
if err != nil {
return err
}
if found && cur.Value.State != "" {
return nil // a wait for a build: the module is not on the machine by it
}
standing := found && cur.Value.Stands()
same := standing && cur.Value.Commit == commit
if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") {
return nil // a send that carries what the machine runs already says nothing new
}
if !resolved {
// The build of the commit is looked up once, and only for a module that moved or names no build.
if build == "" {
if build, err = i.BuildOf(ctx, module, commit); err != nil {
return err
}
}
if build != "" {
if b, found, err := i.BuildByID(ctx, build); err != nil {
return err
} else if found {
published = publishedOf(b.Made)
}
}
resolved = true
}
next := cur.Value
switch {
case !standing:
// Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is
// written over, the ended one staying in history.
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID,
Since: at.UTC()}
case same:
// Already running it: fill what is missing, blank nothing.
if next.Build == "" {
next.Build = build
}
if next.Rollout == "" {
next.Rollout = rolloutID
}
if next.Since.IsZero() {
next.Since = at.UTC()
}
default:
next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC()
next.Artifact, next.Package = "", nil
}
next.SentUnknown = time.Time{}
if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build {
next.Artifact, next.Package = published.Artifact, published.Package
}
if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) &&
next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) {
return nil
}
if !found {
if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) {
continue
} else {
return err
}
}
if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else {
return err
}
}
return fmt.Errorf("the assignment kept moving under the write")
}
// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of
// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto
// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the
// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys
// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended.
// It answers how many keys it wrote; an error is raised by the caller as a condition.
func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) {
c, err := i.theChain(ctx)
if err != nil {
return 0, err
}
rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`)
if err != nil {
return 0, err
}
type kept struct {
name string
sent map[string]string
}
var all []kept
for rows.Next() {
var name string
var raw []byte
if err := rows.Scan(&name, &raw); err != nil {
rows.Close()
return 0, err
}
sent := map[string]string{}
if err := json.Unmarshal(raw, &sent); err != nil {
rows.Close()
return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err)
}
all = append(all, kept{name, sent})
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
now := time.Now().UTC()
for _, k := range all {
if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil {
return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err)
}
if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil {
return 0, err
}
}
return len(all), nil
}
// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that
// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is
// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are.
func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error {
for _, module := range slices.Sorted(maps.Keys(sent)) {
commit := sent[module]
if commit == "" {
continue
}
key := assignmentKey(node, module)
build, err := i.chainBuildOf(ctx, c, module, commit)
if err != nil {
return err
}
for try := 0; ; try++ {
if try == casTries {
return fmt.Errorf("the assignment %s kept moving under the write", key)
}
cur, found, err := c.Assignments.Get(ctx, key)
if err != nil {
return err
}
if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) {
break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown
}
next := cur.Value
if !found {
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at}
}
next.Commit, next.Build = commit, build
if next.Since.IsZero() {
next.Since = at
}
if !found {
_, err = c.Assignments.Create(ctx, key, next)
if errors.Is(err, chain.ErrExists) {
continue
}
} else {
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
if errors.Is(err, chain.ErrMoved) {
continue
}
}
if err != nil {
return err
}
break
}
}
return nil
}
internal/inventory/chain_pending_test.go
-3
View File
@@ -77,9 +77,6 @@ func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) {
if running, _ := inv.Running(ctx, "web"); len(running) != 0 {
t.Fatalf("running %v", running)
}
if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 {
t.Fatalf("runs %v", runs)
}
entries, _ := inv.Catalogued(ctx)
for _, e := range entries {
if e.Manifest.Module == "web" && len(e.On) != 0 {
internal/inventory/chain_rollouts.go
-986
View File
@@ -1,986 +0,0 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan
// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk
// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside.
//
// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value,
// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no
// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the
// merge operations alone; a save keeps what the record holds.
// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the
// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by
// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one.
type rolloutView struct {
chain *chain.Chain
mu sync.RWMutex
entries map[string]viewEntry
ready chan struct{}
once sync.Once
err error
dead atomic.Bool
cancel context.CancelFunc
}
// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out.
type viewEntry struct {
raw []byte
rev uint64
sum planSummary
}
// planSummary is the part of a plan the surveys filter and sort by.
type planSummary struct {
ID string `json:"id"`
State string `json:"state"`
Created time.Time `json:"created"`
Repository string `json:"repository"`
Branch string `json:"branch"`
Merged time.Time `json:"merged"`
Release json.RawMessage `json:"release"`
Held []chain.HeldMerge `json:"held"`
}
func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling }
func (s planSummary) isRelease() bool {
r := strings.TrimSpace(string(s.Release))
return r != "" && r != "null"
}
func newRolloutView(c *chain.Chain) *rolloutView {
return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})}
}
func (v *rolloutView) start() {
ctx, cancel := context.WithCancel(context.Background())
v.cancel = cancel
go func() {
w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx)
if err != nil {
v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err))
return
}
for {
select {
case <-ctx.Done():
v.fail(ctx.Err())
return
case e, open := <-w:
if !open {
v.fail(errors.New("the rollouts bucket stopped answering"))
return
}
if e.Caught {
v.once.Do(func() { close(v.ready) })
continue
}
v.apply(e.Key, e.Value, e.Revision)
}
}
}()
}
// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read.
func (v *rolloutView) fail(err error) {
v.mu.Lock()
if v.err == nil {
v.err = err
}
v.mu.Unlock()
v.dead.Store(true)
v.once.Do(func() { close(v.ready) })
}
func (v *rolloutView) stop() {
if v.cancel != nil {
v.cancel()
}
}
// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one.
func (v *rolloutView) apply(key string, raw []byte, rev uint64) {
var sum planSummary
if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" {
sum = planSummary{ID: key}
}
v.mu.Lock()
defer v.mu.Unlock()
if old, has := v.entries[key]; has && old.rev >= rev {
return
}
v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum}
}
func (v *rolloutView) list() []viewEntry {
v.mu.RLock()
defer v.mu.RUnlock()
out := make([]viewEntry, 0, len(v.entries))
for _, e := range v.entries {
out = append(out, e)
}
return out
}
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
i.state.mu.Lock()
v := i.state.view
if v == nil || v.chain != c || v.dead.Load() {
if v != nil {
v.stop()
}
v = newRolloutView(c)
i.state.view = v
v.start()
}
i.state.mu.Unlock()
select {
case <-v.ready:
case <-ctx.Done():
return nil, ctx.Err()
}
v.mu.RLock()
defer v.mu.RUnlock()
if v.err != nil && len(v.entries) == 0 {
return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err)
}
return v, nil
}
// stopViews ends the view's watch: the inventory is closing.
func (i *Inventory) stopViews() {
i.state.mu.Lock()
defer i.state.mu.Unlock()
if i.state.view != nil {
i.state.view.stop()
i.state.view = nil
}
}
func normalizePlan(p *Plan) {
if p.Modules == nil {
p.Modules = map[string]*PlanModule{}
}
if p.TierEntered.IsZero() {
p.TierEntered = p.Created
}
if !p.Merged.IsZero() {
p.Merged = p.Merged.UTC()
}
}
func planOf(e viewEntry) (Plan, error) {
var p Plan
if err := json.Unmarshal(e.raw, &p); err != nil {
return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err)
}
normalizePlan(&p)
return p, nil
}
func sortCreated(entries []viewEntry) {
sort.SliceStable(entries, func(a, b int) bool {
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
return entries[a].sum.Created.Before(entries[b].sum.Created)
}
return entries[a].sum.ID < entries[b].sum.ID
})
}
func sortCreatedNewest(entries []viewEntry) {
sort.SliceStable(entries, func(a, b int) bool {
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
return entries[a].sum.Created.After(entries[b].sum.Created)
}
return entries[a].sum.ID > entries[b].sum.ID
})
}
// chainPlans is the plans that pass keep, in an order, read from the view.
func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) {
v, err := i.rolloutView(ctx, c)
if err != nil {
return nil, err
}
var chosen []viewEntry
for _, e := range v.list() {
if e.sum.ID != WaitingRecord && keep(e) {
chosen = append(chosen, e)
}
}
order(chosen)
out := make([]Plan, 0, len(chosen))
for _, e := range chosen {
p, err := planOf(e)
if err != nil {
return nil, err
}
out = append(out, p)
}
return out, nil
}
func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) {
v, err := i.rolloutView(ctx, c)
if err != nil {
return Plan{}, false, err
}
var chosen []viewEntry
for _, e := range v.list() {
if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() {
chosen = append(chosen, e)
}
}
sort.SliceStable(chosen, func(a, b int) bool {
if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) {
return chosen[a].sum.Merged.After(chosen[b].sum.Merged)
}
return chosen[a].sum.Created.After(chosen[b].sum.Created)
})
if len(chosen) == 0 {
return Plan{}, false, nil
}
p, err := planOf(chosen[0])
return p, err == nil, err
}
// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view
// learns of it.
func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) {
if !chain.ValidKey(id) {
// No plan has an id the bus takes no spelling of; it is a plain id no plan has.
return Plan{}, fmt.Errorf("no plan %s", id)
}
e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id)
if err != nil {
return Plan{}, err
}
if !found {
return Plan{}, fmt.Errorf("no plan %s", id)
}
if v, err := i.rolloutView(ctx, c); err == nil {
v.apply(id, e.Value, e.Revision)
}
return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}})
}
// ---- the merges
// mergeHolder is a record's copy of a merge.
type mergeHolder struct {
plan string
m chain.HeldMerge
rev uint64
}
func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit }
// mergeHolders is every record's copy of each merge, by the merge's key.
func mergeHolders(entries []viewEntry) map[string][]mergeHolder {
out := map[string][]mergeHolder{}
for _, e := range entries {
for _, m := range e.sum.Held {
k := mergeKey(m.Repository, m.Commit)
out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev})
}
}
return out
}
// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two
// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers
// before one no plan does, then the record written last.
func winner(copies []mergeHolder) mergeHolder {
best := copies[0]
for _, c := range copies[1:] {
switch {
case c.m.Placed.After(best.m.Placed):
best = c
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting:
best = c
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev:
best = c
}
}
return best
}
func winners(entries []viewEntry) map[string]mergeHolder {
out := map[string]mergeHolder{}
for k, copies := range mergeHolders(entries) {
out[k] = winner(copies)
}
return out
}
func batchedOf(h mergeHolder) BatchedMerge {
b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit,
Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone}
if !h.m.Merged.IsZero() {
b.Merged = h.m.Merged.UTC()
}
if h.m.Waiting {
b.Plan = ""
}
return b
}
func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge {
h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(),
Event: m.Event, Alone: m.Alone, Placed: now.UTC()}
if !m.Merged.IsZero() {
h.Merged = m.Merged.UTC()
}
return h
}
func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) {
v, err := i.rolloutView(ctx, c)
if err != nil {
return nil, err
}
return winners(v.list()), nil
}
func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return BatchedMerge{}, false, err
}
h, ok := w[mergeKey(repository, commit)]
if !ok {
return BatchedMerge{}, false, nil
}
return batchedOf(h), true, nil
}
// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard.
func sortMerges(ms []BatchedMerge, newest bool) {
sort.SliceStable(ms, func(a, b int) bool {
x, y := ms[a], ms[b]
switch {
case x.Merged.IsZero() != y.Merged.IsZero():
return !x.Merged.IsZero()
case !x.Merged.Equal(y.Merged):
if newest {
return x.Merged.After(y.Merged)
}
return x.Merged.Before(y.Merged)
}
if newest {
return x.Heard.After(y.Heard)
}
return x.Heard.Before(y.Heard)
})
}
func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if h.plan == plan && !h.m.Waiting {
out = append(out, batchedOf(h))
}
}
sortMerges(out, false)
return out, nil
}
func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) {
out = append(out, batchedOf(h))
}
}
sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) })
return out, nil
}
func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return false, time.Time{}, err
}
var first time.Time
for _, h := range w {
if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) {
first = h.m.Heard
}
}
if first.IsZero() {
return false, time.Time{}, nil
}
return true, first.UTC(), nil
}
// chainEarliestKept is when the controller heard the first merge it holds a record of, on any repository and branch.
func (i *Inventory) chainEarliestKept(ctx context.Context, c *chain.Chain) (bool, time.Time, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return false, time.Time{}, err
}
var first time.Time
for _, h := range w {
if first.IsZero() || h.m.Heard.Before(first) {
first = h.m.Heard
}
}
if first.IsZero() {
return false, time.Time{}, nil
}
return true, first.UTC(), nil
}
func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if h.m.Waiting && h.m.Alone {
out = append(out, batchedOf(h))
}
}
sortMerges(out, true)
return out, nil
}
// chainWaitingMerges is every merge no plan answers that waits and is not walked alone, the newest first.
func (i *Inventory) chainWaitingMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if h.m.Waiting && !h.m.Alone {
out = append(out, batchedOf(h))
}
}
sortMerges(out, true)
return out, nil
}
// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it.
type pendingMerge struct {
plan string
m chain.HeldMerge
}
// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has
// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is
// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3).
func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) {
waiting := m.Plan == ""
if waiting {
// A merge no plan answers is held, flagged, by the record kept for those.
m.Plan = WaitingRecord
if err := i.ensureWaitingRecord(ctx, c); err != nil {
return false, err
}
}
v, err := i.rolloutView(ctx, c)
if err != nil {
return false, err
}
key := mergeKey(m.Repository, m.Commit)
if _, known := winners(v.list())[key]; known {
return false, nil
}
held := heldOf(m, time.Now())
held.Waiting = waiting
found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
for _, x := range h {
if mergeKey(x.Repository, x.Commit) == key {
return h, false
}
}
return append(h, held), true
})
if err != nil {
return false, err
}
if !found {
i.state.mu.Lock()
if i.state.pending == nil {
i.state.pending = map[string]pendingMerge{}
}
i.state.pending[key] = pendingMerge{plan: m.Plan, m: held}
i.state.mu.Unlock()
}
return true, nil
}
// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store
// kept as a row with a null plan. It is no plan: the surveys of plans leave it out.
const WaitingRecord = "waiting-merges"
// ensureWaitingRecord makes the record of the merges no plan answers, once.
func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error {
b := chain.RolloutsOf[Plan](c)
if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found {
return err
}
body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
if err != nil {
return err
}
rev, err := b.CreateRaw(ctx, WaitingRecord, body)
if errors.Is(err, chain.ErrExists) {
return nil
}
if err != nil {
return err
}
if v, verr := i.rolloutView(ctx, c); verr == nil {
v.apply(WaitingRecord, body, rev)
}
return nil
}
// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it.
var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved")
// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again
// when it moved. It answers false when the plan has no record.
func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) {
b := chain.RolloutsOf[Plan](c)
for try := 0; try < casTries; try++ {
e, found, err := b.GetRaw(ctx, id)
if err != nil || !found {
return false, err
}
var record map[string]json.RawMessage
if err := json.Unmarshal(e.Value, &record); err != nil {
return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err)
}
var held []chain.HeldMerge
if raw, has := record["held"]; has {
if err := json.Unmarshal(raw, &held); err != nil {
return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err)
}
}
next, changed := edit(held)
if !changed {
return true, nil
}
if len(next) == 0 {
delete(record, "held")
} else {
raw, err := json.Marshal(next)
if err != nil {
return true, err
}
record["held"] = raw
}
body, err := json.Marshal(record)
if err != nil {
return true, err
}
rev, err := b.UpdateRaw(ctx, id, body, e.Revision)
if errors.Is(err, chain.ErrMoved) {
continue
}
if err != nil {
return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err)
}
if v, verr := i.rolloutView(ctx, c); verr == nil {
v.apply(id, body, rev)
}
return true, nil
}
return true, fmt.Errorf("the merges of plan %s kept moving under the write", id)
}
// recordEdit is what the merges of one record gain, lose and have flagged in one act.
type recordEdit struct {
add []chain.HeldMerge
drop map[string]bool
flag map[string]chain.HeldMerge
}
func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
out := make([]chain.HeldMerge, 0, len(held)+len(e.add))
for _, h := range held {
k := mergeKey(h.Repository, h.Commit)
if e.drop[k] {
continue
}
if f, has := e.flag[k]; has {
h.Waiting, h.Alone = f.Waiting, f.Alone
// A merge put back to waiting may carry a new announcement (HoldMerge); every other flag copies its own.
if len(f.Event) > 0 {
h.Event = f.Event
}
}
out = append(out, h)
}
for _, a := range e.add {
k := mergeKey(a.Repository, a.Commit)
dup := false
for _, h := range out {
if mergeKey(h.Repository, h.Commit) == k {
dup = true
break
}
}
if !dup {
out = append(out, a)
}
}
return out
}
func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 }
// ---- the save
// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save
// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one.
func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
if old == nil {
return now, nil // a new plan enters its first tier now
}
if old.State == PlanAssembling || old.State == PlanQueued {
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
}
since := old.TierEntered
if since.IsZero() {
since = old.Created
}
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
if !wasOpen && p.Open() {
// **A walk set open again enters its tier anew** (issue 1886): a resume or a retry at the tier it ended at. Kept,
// the entry of before its stop measured the tier from hours before the resume — R4e read "waiting … for 5h48m53s"
// seven minutes after it was resumed — and the watchdog of its progress (S3) found it late at once.
return now, nil
}
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
return since, nil // still in the tier, or already ended
}
var modules []string
if old.Tier >= 0 && old.Tier < len(p.Tiers) {
modules = p.Tiers[old.Tier]
}
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State)
if p.Open() {
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier)
}
return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier),
Started: since, Took: now.Sub(since), Detail: detail}
}
// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by
// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is
// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a
// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092).
func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) {
epoch, err := i.actingEpoch(ctx)
if err != nil {
if len(plans) > 0 {
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
}
return err
}
v, err := i.rolloutView(ctx, c)
if err != nil {
return err
}
b := chain.RolloutsOf[Plan](c)
marked := time.Now()
undo := make([]func(), 0, len(plans))
defer func() {
if err != nil {
for _, u := range undo {
u()
}
}
}()
for _, p := range plans {
undo = append(undo, p.markWalk(marked))
}
// Every plan still at the revision it was read at, before anything is written.
cur := make(map[string]chain.Kept[Plan], len(plans))
for _, p := range plans {
k, found, err := b.Get(ctx, p.ID)
if err != nil {
return err
}
if found {
if k.Value.Revision != p.Revision {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
cur[p.ID] = k
}
}
// What each record gains, loses and has flagged.
now := time.Now().UTC()
edits := map[string]*recordEdit{}
editOf := func(id string) *recordEdit {
if edits[id] == nil {
edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}}
}
return edits[id]
}
copies := mergeHolders(v.list())
for _, m := range moves {
key := mergeKey(m.Repository, m.Commit)
held, has := copies[key]
if !has {
// A merge carried for a plan not saved yet goes where it is moved.
i.state.mu.Lock()
if pm, pending := i.state.pending[key]; pending && m.Plan != "" {
pm.plan, pm.m.Alone = m.Plan, m.Alone
i.state.pending[key] = pm
}
i.state.mu.Unlock()
continue
}
w := winner(held)
if m.Plan == "" {
f := w.m
f.Waiting, f.Alone = true, m.Alone
if m.Event != nil {
f.Event = m.Event
}
editOf(w.plan).flag[key] = f
continue
}
if w.plan == m.Plan {
f := w.m
f.Waiting, f.Alone = false, m.Alone
editOf(w.plan).flag[key] = f
continue
}
moved := w.m
moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now
editOf(m.Plan).add = append(editOf(m.Plan).add, moved)
for _, h := range held {
editOf(h.plan).drop[key] = true
}
}
// The records not among the plans: receivers now, givers after.
for id, e := range edits {
if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 {
continue
}
e := e
found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{add: e.add}).apply(h), true
})
if err != nil {
return err
}
if !found {
// Nothing is taken from the record that holds a merge to give it to a plan that has none.
return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add))
}
}
// The plans, those that gain a merge first.
order := append([]*Plan(nil), plans...)
sort.SliceStable(order, func(a, b int) bool {
x, y := edits[order[a].ID], edits[order[b].ID]
return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0)
})
type written struct {
revision int64
entered time.Time
held []chain.HeldMerge
duration *Duration
}
done := map[string]written{}
for _, p := range order {
var w written
for try := 0; ; try++ {
if try == casTries {
return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID)
}
k, found := cur[p.ID]
if try > 0 {
fresh, still, gerr := b.Get(ctx, p.ID)
if gerr != nil {
return gerr
}
if still && fresh.Value.Revision != p.Revision {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
k, found = fresh, still
cur[p.ID] = k
}
record := *p
var old *Plan
var held []chain.HeldMerge
if found {
o := k.Value
old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...)
record.Revision = k.Value.Revision + 1
} else {
record.Revision = 1
}
if e := edits[p.ID]; e != nil {
held = e.apply(held)
}
i.state.mu.Lock()
for key, pm := range i.state.pending {
if pm.plan == p.ID {
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
_ = key
}
}
i.state.mu.Unlock()
record.Held = held
entered, duration := tierLeftOf(old, *p, time.Now())
record.TierEntered, record.Updated = entered, time.Now().UTC()
if epoch != nil {
record.Epoch = uint64(*epoch)
} else {
record.Epoch = 0
}
body, merr := json.Marshal(record)
if merr != nil {
return merr
}
var rev uint64
var werr error
if found {
rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision)
} else {
rev, werr = createRaw(ctx, c, p.ID, body)
}
if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) {
if !found {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
continue
}
if werr != nil {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr)
}
v.apply(p.ID, body, rev)
w = written{revision: record.Revision, entered: entered, held: held, duration: duration}
break
}
done[p.ID] = w
}
// The records that gave a merge up, now it is held by the one it went to.
for id, e := range edits {
if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) {
continue
}
e := e
if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true
}); err != nil {
return err
}
}
i.state.mu.Lock()
for key, pm := range i.state.pending {
if _, saved := done[pm.plan]; saved {
delete(i.state.pending, key)
}
}
i.state.mu.Unlock()
for _, p := range plans {
w := done[p.ID]
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if w.duration != nil {
// After the write: the key of the duration is the plan and its tier, so a measurement made twice is one.
record := i.RecordDuration
if i.recorded != nil {
record = i.recorded
}
if derr := record(ctx, *w.duration); derr != nil {
fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr)
}
}
if PlanSaved != nil {
PlanSaved(i, *p)
}
}
return nil
}
func planNamed(plans []*Plan, id string) bool {
for _, p := range plans {
if p.ID == id {
return true
}
}
return false
}
// createRaw writes a plan's first record.
func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) {
return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body)
}
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
c, err := i.theChain(ctx)
if err != nil {
return 0, err
}
v, err := i.rolloutView(ctx, c)
if err != nil {
return 0, err
}
repaired := 0
for key, copies := range mergeHolders(v.list()) {
if len(copies) < 2 {
continue
}
w := winner(copies)
for _, h := range copies {
if h.plan == w.plan {
continue
}
key := key
if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
}); err != nil {
return repaired, err
}
repaired++
}
}
return repaired, nil
}
internal/inventory/chain_rollouts_test.go
-374
View File
@@ -1,374 +0,0 @@
package inventory
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the
// store but for the durations a save measures, taken here by nothing.
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
t.Helper()
c := chain.InMemory(nil)
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
return inv, c
}
var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
func aBatch(id string, at time.Time) *Plan {
return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling,
Tiers: [][]string{}, Modules: map[string]*PlanModule{}}
}
func aMerge(commit, plan string) BatchedMerge {
return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second),
Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan}
}
// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408
// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first.
func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := context.Background()
p := aBatch("plan-1", plansAt)
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
t.Fatal(err)
}
if p.Revision != 1 {
t.Fatalf("a new plan is at revision %d", p.Revision)
}
stale := *p
p.Note = "moved on"
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 {
t.Fatalf("the second save: revision %d, %v", p.Revision, err)
}
stale.Note = "laid over"
if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) {
t.Fatalf("a stale copy was saved: %v", err)
}
got, err := inv.chainPlanByID(ctx, c, "plan-1")
if err != nil || got.Note != "moved on" || got.Revision != 2 {
t.Fatalf("the plan is %+v: %v", got, err)
}
// A plan new to the bucket that is already there is refused too.
again := aBatch("plan-1", plansAt)
if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) {
t.Fatalf("a new plan over a kept one: %v", err)
}
if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" {
t.Fatalf("a plan nobody kept: %v", err)
}
}
// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record
// after the copy was read is not laid back by the copy's save.
func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := context.Background()
batch := aBatch("plan-1", plansAt)
if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
t.Fatal(err)
}
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added {
t.Fatalf("added %v: %v", added, err)
}
// The copy was read before the merge came in; saving it neither fails nor loses the merge.
batch.Note = "window closed"
if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
t.Fatalf("a copy read before a merge came in: %v", err)
}
merges, err := inv.chainMergesOf(ctx, c, "plan-1")
if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" {
t.Fatalf("the merges of the plan: %+v %v", merges, err)
}
if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" {
t.Fatalf("the merge came back as %+v", merges[0])
}
// A merge heard is not heard twice, whatever the spelling of its repository.
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added {
t.Fatalf("added again %v: %v", added, err)
}
}
// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until
// then it is in no record — so the event that brought it is handed over again if the save fails.
func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := context.Background()
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added {
t.Fatalf("added %v: %v", added, err)
}
if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known {
t.Fatal("a merge for a plan not saved is in a record")
}
// The save is refused: nothing is written, the merge is still carried.
held := false
away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error {
if held {
return nil
}
return errors.New("the lease is not held")
})
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
other.UseChain(away)
t.Cleanup(other.stopViews)
other.state.pending = inv.state.pending
p := aBatch("plan-new", plansAt)
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
t.Fatalf("the save answered %v", err)
}
if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known {
t.Fatal("a refused save left the merge in a record")
}
held = true
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil {
t.Fatal(err)
}
if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" {
t.Fatalf("the save did not write the merge: %+v %v", m, known)
}
}
// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted.
type failingOn struct {
chain.Backend
key string
fail *atomic.Bool
}
func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
if key == f.key && f.fail.Load() {
return 0, errors.New("the bus went away")
}
return f.Backend.Update(ctx, key, value, revision)
}
// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is
// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7).
func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) {
c := chain.InMemory(nil)
var fail atomic.Bool
rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
ctx := context.Background()
a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
for _, p := range []*Plan{a, b} {
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
t.Fatal(err)
}
}
if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
t.Fatal(err)
}
// The move stops before the giving record is written: the merge is in both.
fail.Store(true)
err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}})
if err == nil {
t.Fatal("the move was said to be made with the giving record unwritten")
}
fail.Store(false)
if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" {
t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan)
}
if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
t.Fatalf("the giving plan still answers %+v", got)
}
// The next cut repairs it.
v, _ := inv.rolloutView(ctx, c)
if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
}
repaired := 0
for k, copies := range mergeHolders(v.list()) {
if len(copies) < 2 {
continue
}
w := winner(copies)
for _, h := range copies {
if h.plan != w.plan {
key := k
if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
}); err != nil {
t.Fatal(err)
}
repaired++
}
}
}
if repaired != 1 {
t.Fatalf("repaired %d", repaired)
}
waitFor(t, func() bool {
v, _ := inv.rolloutView(ctx, c)
return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
})
}
// waitFor polls a condition that a watch makes true, for a moment.
func waitFor(t *testing.T, ok func() bool) {
t.Helper()
for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
if ok() {
return
}
}
t.Fatal("the condition did not come true")
}
// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in
// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to.
func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := context.Background()
a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
for _, p := range []*Plan{a, b} {
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
t.Fatal(err)
}
}
if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
t.Fatal(err)
}
if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil {
t.Fatal(err)
}
if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
t.Fatalf("a waiting merge is answered by %+v", got)
}
alone, err := inv.chainAloneMerges(ctx, c)
if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone {
t.Fatalf("waiting: %+v %v", alone, err)
}
if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" {
t.Fatalf("a waiting merge reads as %+v %v", m, known)
}
if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 {
t.Fatalf("a waiting merge is among those a plan answers: %+v", later)
}
if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() {
t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first)
}
// Walked alone: the plan it goes to answers it, and the record that held it lets it go.
if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil {
t.Fatal(err)
}
if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone {
t.Fatalf("plan-b answers %+v", got)
}
if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 {
t.Fatalf("still waiting: %+v", left)
}
waitFor(t, func() bool {
v, _ := inv.rolloutView(ctx, c)
return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
})
// A move to a plan with no record is refused and takes nothing from the one that holds the merge.
err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}})
if !errors.Is(err, ErrMergeUnsaved) {
t.Fatalf("a move to a plan with no record: %v", err)
}
if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 {
t.Fatalf("the merge was taken from plan-b: %+v", got)
}
}
// countingBackend counts the reads a bucket answers.
type countingBackend struct {
chain.Backend
gets, keys atomic.Int32
}
func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
c.gets.Add(1)
return c.Backend.Get(ctx, key)
}
func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
c.keys.Add(1)
return c.Backend.Keys(ctx)
}
// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the
// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view
// through its watch.
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
counting := &countingBackend{Backend: chain.NewMemory()}
c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
ctx := context.Background()
for k, id := range []string{"plan-1", "plan-2", "plan-3"} {
p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute))
if k > 0 {
p.State = PlanBuilding
}
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
t.Fatal(err)
}
}
before := counting.gets.Load()
for n := 0; n < 20; n++ {
open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
if err != nil || len(open) != 2 || open[0].ID != "plan-2" {
t.Fatalf("open plans %+v: %v", open, err)
}
if _, err := inv.chainAloneMerges(ctx, c); err != nil {
t.Fatal(err)
}
}
if counting.gets.Load() != before || counting.keys.Load() != 0 {
t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
}
// Another process writes; the view follows.
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
other.UseChain(c)
t.Cleanup(other.stopViews)
late := aBatch("plan-4", plansAt.Add(time.Hour))
late.State = PlanRolling
if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil {
t.Fatal(err)
}
waitFor(t, func() bool {
open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
return len(open) == 3
})
}
// Where the record of merges begins is the earliest merge kept on any repository and branch (novox/hq ADR 0463 §2).
func TestTheRecordOfMergesBeginsAtTheEarliestMergeKeptOnAnyTrunk(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := context.Background()
if kept, _, err := inv.chainEarliestKept(ctx, c); err != nil || kept {
t.Fatalf("an inventory with no merge kept says it keeps one: %v %v", kept, err)
}
if err := inv.chainSavePlans(ctx, c, []*Plan{aBatch("plan-1", plansAt), aBatch("plan-2", plansAt)}, nil); err != nil {
t.Fatal(err)
}
later := aMerge("m-late", "plan-1")
later.Repository, later.Branch, later.Heard = "novox/mesh-sdk", "main", plansAt.Add(48*time.Hour)
earlier := aMerge("m-early", "plan-2")
earlier.Repository, earlier.Branch, earlier.Heard = "novox/photos", "nox-mesh", plansAt.Add(time.Hour)
for _, m := range []BatchedMerge{later, earlier} {
if added, err := inv.chainAddMerge(ctx, c, m); err != nil || !added {
t.Fatalf("%s: %v %v", m.Commit, added, err)
}
}
kept, first, err := inv.chainEarliestKept(ctx, c)
if err != nil || !kept || !first.Equal(plansAt.Add(time.Hour)) {
t.Fatalf("the record begins at %v (%v, %v), want %v", first, kept, err, plansAt.Add(time.Hour))
}
}
internal/inventory/chain_sent_copy_test.go
-111
View File
@@ -1,111 +0,0 @@
package inventory
import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review
// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy
// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build
// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build.
func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := t.Context()
at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC)
if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil {
t.Fatal(err)
}
put := func(module string, a chain.Assignment) {
a.Node, a.Module = "laptop", module
if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil {
t.Fatal(err)
}
}
put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet
put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one
put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since
put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build
put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned
sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand",
"waits": "c-waits", "ended": "c-ended"}
for pass := 0; pass < 2; pass++ { // and the second pass changes nothing
if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil {
t.Fatal(err)
}
get := func(module string) chain.Assignment {
t.Helper()
k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module))
if err != nil || !found {
t.Fatalf("%s: %v %v", module, found, err)
}
return k.Value
}
if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn {
t.Errorf("notes: %+v", a)
}
if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" {
t.Errorf("a key with a commit was overwritten: %+v", a)
}
if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() {
t.Errorf("a hand send's unknown was overwritten: %+v", a)
}
if a := get("waits"); a.Commit != "" || a.State != chain.StatePending {
t.Errorf("a wait was overwritten: %+v", a)
}
if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" {
t.Errorf("the pulled-in module has no standing key of its own: %+v", a)
}
if a := get("ended"); a.Commit != "" || a.Stands() {
t.Errorf("an ended key was revived or given a commit: %+v", a)
}
}
}
// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the
// pulled-in module, and a second copy finds nothing to do.
func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`)
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 {
t.Fatalf("the copy did %d nodes (%v)", n, err)
}
got, known, err := inv.SentBuilds(ctx, "laptop")
if err != nil || !known || got["mesh-wireguard"] != "c-wg" {
t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err)
}
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
t.Fatalf("the column was not emptied: %v", rows)
}
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
t.Fatalf("a second copy did %d nodes (%v)", n, err)
}
}
// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared
// without a commit.
func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
record, err := inv.AddNode(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`)
if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil {
t.Fatal(err)
}
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
t.Fatalf("a send left the column: %v", rows)
}
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
t.Fatalf("the copy did %d nodes (%v)", n, err)
}
}
internal/inventory/chain_sent_test.go
-116
View File
@@ -1,116 +0,0 @@
//go:build integration
package inventory
import (
"context"
"errors"
"reflect"
"testing"
"github.com/novox/mesh-controller/internal/chain"
)
// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned
// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state.
func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) {
inv, node := aNodeWithModules(t, "web", "cache")
ctx := t.Context()
record, _ := inv.NodeByName(ctx, node)
if _, err := inv.Assign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) {
t.Fatalf("sent %v %v %v", builds, known, err)
}
pulled, found, _ := inv.AssignmentValue(ctx, node, "cache")
if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" {
t.Fatalf("the module only carried has the key %+v", pulled)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) {
t.Fatalf("a module only pulled in is assigned: %v", on)
}
// Assigned later, it keeps what it was sent and is pulled in no longer.
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
t.Fatal(err)
}
made, _, _ := inv.AssignmentValue(ctx, node, "cache")
if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" {
t.Fatalf("assigned over a pulled-in key: %+v", made)
}
}
// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds**
// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so.
func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) {
inv, node := aNodeWithModules(t, "web", "cache")
ctx := t.Context()
record, _ := inv.NodeByName(ctx, node)
if _, err := inv.Assign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil {
t.Fatal(err)
}
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) {
t.Fatalf("the second send did not replace the map: %v %v", builds, known)
}
if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" {
t.Fatalf("a module no longer carried keeps %+v", old)
}
// By hand: unknown, with a time and no commit anywhere.
if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil {
t.Fatal(err)
}
if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil {
t.Fatalf("a hand send reads as %v %v", builds, known)
}
web, _, _ := inv.AssignmentValue(ctx, node, "web")
if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() {
t.Fatalf("web after a hand send: %+v", web)
}
if sent, _ := inv.Outstanding(ctx, node); sent != "d3" {
t.Fatalf("the digest of the hand send: %q", sent)
}
// A send that names its builds makes it known again.
if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil {
t.Fatal(err)
}
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) {
t.Fatalf("after a send that names its builds: %v %v", builds, known)
}
if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() {
t.Fatalf("web still says unknown: %+v", again)
}
}
// failingToWrite is an assignments bucket that fails every write.
type failingToWrite struct{ chain.Backend }
func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) {
return 0, errors.New("the bus went away")
}
func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) {
return 0, errors.New("the bus went away")
}
// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the
// keys, so a send recorded without them would read as one that never happened.
func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) {
inv, node := aNodeWithModules(t, "web")
ctx := t.Context()
record, _ := inv.NodeByName(ctx, node)
inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()},
func(context.Context) error { return nil }))
err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"})
if err == nil {
t.Fatal("a send whose links could not be written was recorded")
}
}
internal/inventory/chain_store_test.go
+42 -28
View File
@@ -4,6 +4,7 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"reflect"
"slices"
@@ -72,10 +73,6 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
if err != nil || len(announced) != 3 {
t.Fatalf("announceable %d: %v", len(announced), err)
}
prints, err := inv.Fingerprints(ctx)
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
t.Fatalf("fingerprints %v: %v", prints, err)
}
}
// A build request row as the store held it before the move.
@@ -166,7 +163,7 @@ func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T)
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
// may be forgotten, what is behind — read the bucket, and a seat held by the
// assignment goes with it.
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
inv, node := aNodeWithModules(t, "postgres", "web")
@@ -174,7 +171,6 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
start := generationNow(t, inv)
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
t.Fatalf("assigned %v: %v", added, err)
}
@@ -185,8 +181,24 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
}
if generationNow(t, inv) <= start {
t.Fatal("an assignment left the generation where it was")
// **Placement only** (ADR 0539 §12): the value on the bus names the node, the module and since when, as mesh-config
// reads them (unknown fields ignored there), and no build, commit or rollout.
kv, err := inv.KeptAssignments(ctx)
if err != nil || len(kv["anchor.postgres"].Value) == 0 {
t.Fatalf("the kept assignments: %v %v", kv, err)
}
var read struct {
Node string `json:"node"`
Module string `json:"module"`
Assigned time.Time `json:"assigned"`
}
if err := json.Unmarshal(kv["anchor.postgres"].Value, &read); err != nil || read.Node != "anchor" || read.Module != "postgres" || read.Assigned.IsZero() {
t.Fatalf("a reader of placement reads %+v from %s: %v", read, kv["anchor.postgres"].Value, err)
}
for _, not := range []string{`"build"`, `"commit"`, `"rollout"`, `"artifact"`, `"package"`, `"since"`} {
if strings.Contains(string(kv["anchor.postgres"].Value), not) {
t.Errorf("an assignment names %s: %s", not, kv["anchor.postgres"].Value)
}
}
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
t.Fatalf("a module the mesh does not know: %v", err)
@@ -224,13 +236,9 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
t.Fatalf("a seat handed to an assignment: %v", err)
}
before := generationNow(t, inv)
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
if generationNow(t, inv) <= before {
t.Fatal("an unassignment left the generation where it was")
}
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
@@ -278,12 +286,12 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
}
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// plan of a module the machine was sent is written into its assignment, another field of the value stays,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// plan of a module the machine was sent is written into its assignment, another field of the value stays,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
@@ -300,23 +308,14 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
t.Fatal(err)
}
// A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this
// one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded.
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
t.Fatal(err)
}
sentPlans(t, inv, record.ID, raw)
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
written, removed, err := inv.KeepAssignmentPlans(ctx)
if err != nil || written != 2 || removed != 0 {
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
}
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
if dunst.Value.Provisioning == nil || dunst.Value.Assigned.IsZero() {
t.Fatalf("dunst's value is %+v", dunst.Value)
}
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
@@ -334,9 +333,7 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi
}
// Sent again without the pulled-in module: its value ends, the assignments stay.
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
t.Fatal(err)
}
sentPlans(t, inv, record.ID, only)
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
t.Fatalf("ended %d: %v", removed, err)
}
@@ -351,3 +348,20 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
// already is left as it is.
// sentPlans gives a machine the plans it was last sent, as the column the mirror reads keeps them.
func sentPlans(t *testing.T, inv *Inventory, id string, plans []byte) {
t.Helper()
if _, err := inv.store.Pool().Exec(t.Context(), `update node set sent_provisioning = $2 where id = $1`, id, plans); err != nil {
t.Fatal(err)
}
}
func mustJSON(t *testing.T, v any) string {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
internal/inventory/collection.go
+2 -22
View File
@@ -458,7 +458,7 @@ func keptLately(builds []Build, planned map[string]bool, now time.Time, within t
return out
}
// keptLatelyOrPlanned is keptLately of every build the records hold, against the plans that have not ended.
// keptLatelyOrPlanned is keptLately of every build the records hold, (no plan names a build any more, ADR 0539 §17).
func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]string, error) {
c, err := i.theChain(ctx)
if err != nil {
@@ -468,27 +468,7 @@ func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]strin
if err != nil {
return nil, err
}
plans, err := i.OpenPlans(ctx)
if err != nil {
return nil, err
}
return keptLately(all, plannedBuilds(plans), time.Now().UTC(), i.lately()), nil
}
// plannedBuilds is every build id these plans name: a module's build, and the builds a rollout carried.
func plannedBuilds(plans []Plan) map[string]bool {
out := map[string]bool{}
for _, p := range plans {
for _, m := range p.Modules {
if m != nil && m.Build != "" {
out[m.Build] = true
}
}
for _, id := range p.Builds {
out[id] = true
}
}
return out
return keptLately(all, nil, time.Now().UTC(), i.lately()), nil
}
// digestIn is the `sha256:<hex>` a reference names, empty when it names none.
internal/inventory/durations.go
-134
View File
@@ -1,134 +0,0 @@
package inventory
import (
"context"
"time"
)
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
// The kinds of duration recorded.
const (
DurationApply = "apply"
DurationPlanTier = "plan-tier"
DurationBuild = "build"
// DurationWalkPhase is one phase of an ended walk, per class (novox/hq ADR 0282 decision 6): subject
// "<class>/<phase>", and "<class>/total" for its merge to every machine running it.
DurationWalkPhase = "walk-phase"
// DurationBusy is a machine's wait for a busy container (novox/hq ADR 0335): from its first report of a
// send, which said a container waits because it is busy, to the report that says it replaced — the same
// send's ref as its `apply` row, whose detail is ApplyBusy. Its own measure, never in the apply's.
DurationBusy = "busy-wait"
)
// ApplyBusy is the detail of an `apply` duration whose report said a container waits because it is busy: the
// machine answered, and is not yet running what it was sent (novox/hq ADR 0335).
const ApplyBusy = "busy"
// DurationKinds are every kind, in the order `durations` shows them.
var DurationKinds = []string{DurationApply, DurationPlanTier, DurationBuild, DurationWalkPhase,
DurationBusy}
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
// in Phase 1's first live week.
const DurationsKeptFor = 30 * 24 * time.Hour
// Duration is one measurement.
type Duration struct {
Kind string `json:"kind"`
Subject string `json:"subject"`
Node string `json:"node,omitempty"`
Ref string `json:"ref"`
Started time.Time `json:"started"`
Took time.Duration `json:"took"`
Detail string `json:"detail,omitempty"`
}
// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
if d.Took < 0 {
return nil // a clock that went back measures nothing
}
_, err := i.store.Pool().Exec(ctx,
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
return err
}
// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
// to the first report of it. A report of anything else, or of a send already measured, measures
// nothing — a machine reconciling reports the same declaration every few minutes.
func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
if declared == "" {
return nil
}
_, err := i.store.Pool().Exec(ctx,
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
from node n
where n.name = $2 and n.sent = $3 and n.sent_at is not null
on conflict do nothing`,
DurationApply, node, declared, outcome)
return err
}
// RecordBusyDuration measures a machine's wait for a busy container, at the report that says it replaced
// (novox/hq ADR 0335): from the end of its `apply` duration of the send it was last sent, which said it waited, to
// now. Nothing when that send's first report did not say a wait, and once per send.
func (i *Inventory) RecordBusyDuration(ctx context.Context, node, declared string) error {
if declared == "" {
return nil
}
_, err := i.store.Pool().Exec(ctx,
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
select $1, d.subject, d.node, d.ref, d.started + d.took_ms * interval '1 millisecond',
greatest(0, (extract(epoch from (now() - (d.started + d.took_ms * interval '1 millisecond'))) * 1000)::bigint), ''
from duration d join node n on n.name = d.node
where d.kind = $2 and d.subject = $3 and d.detail = $5 and n.sent = $4 and n.sent_at is not null
and d.ref = n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US')
on conflict do nothing`,
DurationBusy, DurationApply, node, declared, ApplyBusy)
return err
}
// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
rows, err := i.store.Pool().Query(ctx,
`select kind, subject, node, ref, started, took_ms, detail from duration
where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Duration
for rows.Next() {
var d Duration
var ms int64
if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
return nil, err
}
d.Took = time.Duration(ms) * time.Millisecond
out = append(out, d)
}
return out, rows.Err()
}
// WalkPhasesKept says whether any phase of a walk is kept as a walk-phase duration, and whether its total is.
func (i *Inventory) WalkPhasesKept(ctx context.Context, plan string) (anyKept, totalKept bool, err error) {
err = i.store.Pool().QueryRow(ctx,
`select count(*) > 0, count(*) filter (where ref = $2) > 0 from duration where kind = $1 and ref like $3`,
DurationWalkPhase, plan+"/total", plan+"/%").Scan(&anyKept, &totalKept)
return anyKept, totalKept, err
}
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
time.Now().Add(-DurationsKeptFor))
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
internal/inventory/durations_test.go
-80
View File
@@ -1,80 +0,0 @@
//go:build integration
package inventory
import (
"testing"
"time"
)
// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
// send measured at its first report and not again at every reconcile that repeats it; a send of
// something else measures nothing; a new send is a new measurement.
func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
node, err := inv.AddNode(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
t.Fatal(err)
}
for range 3 { // the report, then two reconciles saying the same
if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
t.Fatal(err)
}
ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
t.Fatalf("%v %+v", err, ds)
}
time.Sleep(10 * time.Millisecond)
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
t.Fatal(err)
}
if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
t.Fatal(err)
}
if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
t.Fatalf("a second send was not measured: %+v", ds)
}
}
// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
p.State = PlanRolling // the same tier, saved again
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
t.Fatalf("a tier not left was measured: %+v", ds)
}
p.Tier = 1
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
p.State = PlanDone
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure
t.Fatal(err)
}
ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
ds[1].Ref != "plan-1/tier-1" {
t.Fatalf("%v %+v", err, ds)
}
}
internal/inventory/gate.go
-335
View File
@@ -1,335 +0,0 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The gate's verdicts (novox/hq ADR 0236, to-be 45 §8): what a build did on its first machine, and,
// for one that failed there, how it was put back. One row per build, written by the plan that rolled it
// out, under the lease.
// The gate's verdicts and a failed build's rollback.
const (
GatePassed = "passed"
GateFailed = "failed"
// GateSuperseded is a judging ended by a later send to the judged machine that moved the module to
// another build (novox/hq issue 352): no verdict on the build, and nothing put back.
GateSuperseded = "superseded"
RollingBack = "rolling-back"
RolledBack = "rolled-back"
NotRolledBack = "not-rolled-back"
)
// GateVerdict is one build's verdict at its gate.
type GateVerdict struct {
Build string
Module string
Commit string
Previous string
Plan string
Machines []string
Verdict string
Rollback string
Why string
Component string
// JudgingFrom is when the first machine reported the build applied and the judging began.
JudgingFrom *time.Time
JudgedAt time.Time
Epoch uint64
}
// RecordGate writes a build's verdict. A build that passed on one machine may still fail on the next one
// judged; **a failed build's row is written once**: any later verdict for it is refused with ErrGateKept, which is what keeps a rollback to one per build — the row
// is written before the rollback's send, and a controller replaced in between finds it.
func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error {
epoch, err := i.actingEpoch(ctx)
if err != nil {
return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err)
}
if v.Machines == nil {
v.Machines = []string{}
}
tag, err := i.store.Pool().Exec(ctx,
`insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why,
component, judging_from, judged_at, epoch)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12)
on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback,
why = excluded.why, previous = excluded.previous, machines = excluded.machines,
judged_at = now(), epoch = excluded.epoch
where build_gate.verdict = 'passed'`,
v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component,
v.JudgingFrom, epoch)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return fmt.Errorf("%w: %s", ErrGateKept, v.Build)
}
return nil
}
// ErrGateKept is a verdict already kept for the build, which is not written over.
var ErrGateKept = errors.New("this build's verdict at its gate is already kept")
// SetRollback records how a failed build's rollback went.
func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error {
if _, err := i.actingEpoch(ctx); err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx,
`update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`,
build, rollback, why)
return err
}
// GateOf is a build's verdict, and whether it has one.
func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) {
rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build)
if err != nil {
return GateVerdict{}, false, err
}
list, err := scanGates(rows)
if err != nil || len(list) == 0 {
return GateVerdict{}, false, err
}
return list[0], true, nil
}
// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own.
func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) {
v, found, err := i.GateOf(ctx, build)
return found && v.Verdict == GateFailed, err
}
// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads.
func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) {
rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan,
machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0)
from build_gate order by module, judged_at desc`)
if err != nil {
return nil, err
}
return scanGates(rows)
}
// Gates is the newest verdicts, newest first: what `plans gates` lists.
func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) {
rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit)
if err != nil {
return nil, err
}
return scanGates(rows)
}
const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component,
judging_from, judged_at, coalesce(epoch, 0) from build_gate`
func scanGates(rows pgx.Rows) ([]GateVerdict, error) {
defer rows.Close()
var out []GateVerdict
for rows.Next() {
var v GateVerdict
var epoch int64
if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict,
&v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil {
return nil, err
}
v.Epoch = uint64(epoch)
out = append(out, v)
}
return out, rows.Err()
}
// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build
// that worked, made from the commit the first machine ran before, asked before the failed one, and not
// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one
// may already be gone from the artifact store. False when there is none to go back to.
func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) {
builds, err := i.Builds(ctx, module, 50)
if err != nil {
return Build{}, false, err
}
kept := 0
for _, b := range builds {
if !b.Worked() {
continue
}
kept++
if kept > KeptBuilds {
break
}
if b.ID == failed.ID || (commit != "" && b.Commit != commit) {
continue
}
// Only a build of the repository the failed build was made from — the module's, since its take-in
// registered it (novox/hq ADR 0266): a build of the module's name from another repository is recorded
// and was never registered, and putting it back would register it now.
if failed.Repository != "" && !sameRepositoryAs(b.Repository, failed.Repository) {
continue
}
if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) {
continue
}
if bad, err := i.GateFailed(ctx, b.ID); err != nil {
return Build{}, false, err
} else if bad {
continue
}
manifest, err := i.manifestOf(ctx, b.ID)
if err != nil {
return Build{}, false, err
}
if len(manifest) == 0 || string(manifest) == "null" {
continue
}
b.Manifest = manifest
return b, true, nil
}
return Build{}, false, nil
}
// sameRepositoryAs says two recorded repositories are one, however their case or `.git` is spelled.
func sameRepositoryAs(a, b string) bool {
trim := func(s string) string {
return strings.TrimSuffix(strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/"), ".git")
}
return trim(a) == trim(b)
}
// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it
// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can
// never register over it again (issue 219's order). The source's head is left where the merge moved it:
// the module IS behind its source, and `status` says so.
func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
if _, err := i.actingEpoch(ctx); err != nil {
return fmt.Errorf("%s is not put back: %w", b.Module, err)
}
m, err := catalogue.ParseManifest(b.Manifest)
if err != nil {
return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err)
}
raw, err := json.Marshal(m)
if err != nil {
return err
}
tag, err := i.store.Pool().Exec(ctx,
`update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''),
built_asked = now(), registered = now()
where name = $1`, b.Module, raw, m.Version, b.Commit)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module)
}
return nil
}
// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
// from one that does (the bus's planned step, ADR 0236).
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainBuildFingerprints(ctx, c, module)
}
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainFingerprints(ctx, c)
}
// PassedCommits is, per module, the commits a build of which passed its gate on some machine.
func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bool, error) {
rows, err := i.store.Pool().Query(ctx, `select module, commit_hash from build_gate where verdict = 'passed'`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]map[string]bool{}
for rows.Next() {
var module, commit string
if err := rows.Scan(&module, &commit); err != nil {
return nil, err
}
if out[module] == nil {
out[module] = map[string]bool{}
}
out[module][commit] = true
}
return out, rows.Err()
}
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
// recorded (a manifest handed over by hand).
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
c, err := i.theChain(ctx)
if err != nil {
return "", err
}
return i.chainBuildOf(ctx, c, module, commit)
}
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
// newest successful build from it, with the artifacts of the build standing for it when its source was
// unchanged (issue 280) — what a machine last sent that build runs. False when no such build, or none
// with a manifest, is kept (novox/hq issue 295: a recorded module is composed at the build its machine
// runs until a person's push moves it).
func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (catalogue.Manifest, bool, error) {
if commit == "" {
return catalogue.Manifest{}, false, nil
}
id, raw, found, err := i.newestManifestAt(ctx, module, commit)
if err != nil {
return catalogue.Manifest{}, false, err
}
if !found {
return catalogue.Manifest{}, false, nil
}
if stands, same, err := i.StandingBuild(ctx, module, id); err != nil {
return catalogue.Manifest{}, false, err
} else if stands != "" && stands != id && len(same) > 0 {
raw = same
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
return catalogue.Manifest{}, false, fmt.Errorf("%s's build from %s is not a manifest the mesh can compose: %w",
module, commit, err)
}
return m, true, nil
}
// manifestOf is the manifest a build was recorded with; nil when it kept none.
func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainManifestOf(ctx, c, id)
}
// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest:
// its id and manifest; false when there is none.
func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) {
c, err := i.theChain(ctx)
if err != nil {
return "", nil, false, err
}
return i.chainManifestBuild(ctx, c, module, commit)
}
internal/inventory/heartbeat_gap_gone_test.go
-12
View File
@@ -1,12 +0,0 @@
package inventory
import "testing"
// 1183/AC3: the heartbeat-gap duration is no kind the mesh measures.
func TestNoHeartbeatGapIsMeasured(t *testing.T) { // 1183/AC3
for _, k := range DurationKinds {
if k == "heartbeat-gap" {
t.Fatalf("%s is still a kind of duration: %v", k, DurationKinds)
}
}
}
internal/inventory/heldhere_test.go
-32
View File
@@ -1,32 +0,0 @@
package inventory
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
)
// novox/hq issue 218: a seat held once for the mesh is granted and issued only to the holder on record;
// a node seat to every machine's claimant; a mesh seat with no holder on record as derived.
func TestOnlyTheRecordedHolderHoldsAMeshSeat(t *testing.T) {
claimed := []broker.Seat{
{Name: "mesh-store", Scope: catalogue.ScopeMesh},
{Name: "node-packet-filter", Scope: catalogue.ScopeNode},
{Name: "unrecorded", Scope: catalogue.ScopeMesh},
}
holdings := []catalogue.Held{{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "control", Module: "postgres"}}
names := func(ss []broker.Seat) (out []string) {
for _, s := range ss {
out = append(out, s.Name)
}
return
}
if got := names(heldHere(claimed, holdings, "control", "postgres")); len(got) != 3 {
t.Errorf("the holder lost a seat: %v", got)
}
got := names(heldHere(claimed, holdings, "other", "postgres"))
if len(got) != 2 || got[0] != "node-packet-filter" || got[1] != "unrecorded" {
t.Errorf("a claimant on another machine holds %v; want the node seat and the unrecorded one, not the store", got)
}
}
internal/inventory/hold.go
-149
View File
@@ -1,149 +0,0 @@
package inventory
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"sync"
"time"
)
// ErrNodeBusy is a node another act kept holding for longer than a caller waits.
var ErrNodeBusy = errors.New("another act is composing or sending that node's declaration")
// HoldWaitFor is how long HoldNodes waits for a node another act holds, and HoldPoll how often it
// looks again. Variables so a test need not wait minutes.
var (
HoldWaitFor = 2 * time.Minute
HoldPoll = 250 * time.Millisecond
)
// HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while
// one caller holds a node, another asking for it waits. Without it a push that composed a node as
// adopted could send that declaration after `converge --yes` sent the converged one, and the node
// would return to adopted with nobody having asked.
//
// Session-level advisory locks on one connection, all or none: a set not wholly free is given back
// at once, so two callers holding overlapping sets never each wait on the other. **A waiter pins
// no connection.** It looks again every HoldPoll with a connection borrowed for the look, and gives
// up after HoldWaitFor with ErrNodeBusy naming the node — so a stuck holder costs the pool one
// connection, never one per caller queued behind it. Release gives every one back, and may be
// called more than once.
func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) {
sorted := slices.Clone(names)
slices.Sort(sorted)
sorted = slices.Compact(sorted)
deadline := time.Now().Add(HoldWaitFor)
for {
release, busy, err := i.tryHold(ctx, sorted)
if err != nil || busy == "" {
return release, err
}
if time.Now().After(deadline) {
return nil, fmt.Errorf("%w: %s has been held for over %s — try again once it is done",
ErrNodeBusy, busy, HoldWaitFor)
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(HoldPoll):
}
}
}
// tryHold takes every named node's lock or none, and says which node was busy when it took none.
func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), string, error) {
conn, err := i.store.Pool().Acquire(ctx)
if err != nil {
return nil, "", err
}
var once sync.Once
release := func() {
once.Do(func() {
// Unlocking all of this session's advisory locks, then handing the connection back: a
// connection returned still holding one would hold it for whoever borrows it next.
_, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`)
if err != nil {
// The session's locks die with the session: close it rather than return it.
_ = conn.Conn().Close(context.WithoutCancel(ctx))
}
conn.Release()
})
}
for _, name := range sorted {
var took bool
if err := conn.QueryRow(ctx,
`select pg_try_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`,
name).Scan(&took); err != nil {
release()
return nil, "", err
}
if !took {
release()
return nil, strings.TrimSpace(name), nil
}
}
return release, "", nil
}
// ErrPlansBusy is the plans held by another act — on a machine replacing its controller, the other
// controller — for longer than a caller waits, or at all for one that does not wait.
var ErrPlansBusy = errors.New("another controller is working the plans")
// HoldPlans makes working the plans one act at a time, across every controller on the store
// (novox/hq issue 213). A plan is read, changed and written whole; two controllers doing that at
// once — the old and the new for the moment a machine hands its controller over, or a controller
// and a person's `plans stop` — each act on what the other has not saved yet: a tier asked twice,
// an outcome written over. A session-level advisory lock on one connection, released by the
// returned function and by the session ending, so a controller that dies holding it holds nothing.
//
// wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is
// moving the plans already; and a build outcome's on the event loop, which the build records keep
// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to
// HoldWaitFor — a merge's way, which must be written.
func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) {
deadline := time.Now().Add(HoldWaitFor)
for {
release, took, err := i.tryLock(ctx, "mesh-plans")
if err != nil || took {
return release, err
}
if !wait || time.Now().After(deadline) {
return nil, ErrPlansBusy
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(HoldPoll):
}
}
}
// tryLock takes one named advisory lock on a connection of its own, or gives the connection back.
func (i *Inventory) tryLock(ctx context.Context, key string) (func(), bool, error) {
conn, err := i.store.Pool().Acquire(ctx)
if err != nil {
return nil, false, err
}
var once sync.Once
release := func() {
once.Do(func() {
if _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`); err != nil {
_ = conn.Conn().Close(context.WithoutCancel(ctx))
}
conn.Release()
})
}
var took bool
if err := conn.QueryRow(ctx, `select pg_try_advisory_lock(hashtext($1)::bigint)`, key).Scan(&took); err != nil {
release()
return nil, false, err
}
if !took {
release()
return nil, false, nil
}
return release, true, nil
}
internal/inventory/hold_plans_test.go
-40
View File
@@ -1,40 +0,0 @@
//go:build integration
package inventory
import (
"errors"
"testing"
"time"
)
// novox/hq issue 213: while a machine hands its controller over from the container to the process,
// two controllers run on one store for a moment. Working the plans is one act at a time across them.
func TestThePlansAreWorkedByOneControllerAtATime(t *testing.T) {
first := ForTest(t)
// A second controller: its own connections to the same store.
second, err := Open(t.Context())
if err != nil {
t.Fatal(err)
}
t.Cleanup(second.Close)
release, err := first.HoldPlans(t.Context(), false)
if err != nil {
t.Fatalf("the plans could not be held when nobody held them: %v", err)
}
t.Cleanup(release) // a pool closing waits for a connection still held; release is idempotent
if _, err := second.HoldPlans(t.Context(), false); !errors.Is(err, ErrPlansBusy) {
t.Fatalf("a second controller held the plans while the first did: %v", err)
}
// A waiter gets them once they are let go.
was := HoldPoll
HoldPoll = 10 * time.Millisecond
defer func() { HoldPoll = was }()
go func() { time.Sleep(50 * time.Millisecond); release() }()
again, err := second.HoldPlans(t.Context(), true)
if err != nil {
t.Fatalf("a waiting controller never got the plans once they were let go: %v", err)
}
again()
}
internal/inventory/hold_test.go
-93
View File
@@ -1,93 +0,0 @@
//go:build integration
package inventory
import (
"errors"
"strings"
"testing"
"time"
)
// Composing and sending one node's declaration is serialised: a second holder waits for the first.
func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
release, err := inv.HoldNodes(ctx, []string{"anchor", "laptop"})
if err != nil {
t.Fatal(err)
}
got := make(chan func(), 1)
go func() {
second, err := inv.HoldNodes(ctx, []string{"laptop"})
if err != nil {
t.Error(err)
got <- func() {}
return
}
got <- second
}()
select {
case <-got:
t.Fatal("a node held by one caller was held by another at the same time")
case <-time.After(300 * time.Millisecond):
}
// Another node is not held up.
other, err := inv.HoldNodes(ctx, []string{"joiner"})
if err != nil {
t.Fatal(err)
}
other()
release()
select {
case second := <-got:
second()
case <-time.After(5 * time.Second):
t.Fatal("releasing the node did not let the next holder in")
}
}
// A waiter pins no pool connection while it waits, and gives up after a bounded wait saying which
// node is busy.
func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
release, err := inv.HoldNodes(ctx, []string{"anchor"})
if err != nil {
t.Fatal(err)
}
defer release()
savedWait, savedPoll := HoldWaitFor, HoldPoll
HoldWaitFor, HoldPoll = 1500*time.Millisecond, 50*time.Millisecond
defer func() { HoldWaitFor, HoldPoll = savedWait, savedPoll }()
pool := inv.store.Pool()
base := pool.Stat().AcquiredConns()
const waiters = 3
done := make(chan error, waiters)
for range waiters {
go func() {
_, err := inv.HoldNodes(ctx, []string{"anchor"})
done <- err
}()
}
// While they wait, the pool lends nothing to them for longer than a look: **pinned is held throughout**,
// so at some moment of the wait fewer than all of them hold one. Asked of the least seen over many
// looks, not of the most: a look that lands while every waiter is mid-query is a busy machine, not a
// pinned connection, and on a loaded one (the suite's packages at once, the race detector) it did land.
least := waiters
for range 100 {
time.Sleep(10 * time.Millisecond)
if n := int(pool.Stat().AcquiredConns() - base); n < least {
least = n
}
}
if least >= waiters {
t.Fatalf("%d waiters held %d connections at every look across their wait", waiters, least)
}
for range waiters {
if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") {
t.Fatalf("a waiter did not give up naming the busy node: %v", err)
}
}
}
internal/inventory/migrations/0108-the-plans-leave-the-store.sql
+16
View File
@@ -0,0 +1,16 @@
-- The tables of the delivery plans' machinery leave the store (novox/hq ADR 0539 §17, design 47 release 6, issue 1945).
--
-- A machine follows its wants and applies a module from the composition the controller writes for it (ADR 0539 §1, §2), so
-- the controller keeps no record of what it sent, no counter of the assignments a send was composed from, no verdict of a
-- gate that judged a build on its first machine and no measure of how long a plan's tier or a machine's apply took. This
-- drops the four tables that kept them: declaration_send (0094), assignment_generation (0094), build_gate (0073) and duration
-- (0066, 0075).
--
-- **It is one-way**: a controller older than this schema is refused by the put-back gate (the highest migration it carries is
-- recorded by its version, issue 352). What stays: the node row and its columns (the digest of the last send, which the old
-- reads of `status` still name until they go with their verbs), the bus step, the core box and the refused registrations.
-- The function raise_assignment_generation and its triggers went with 0105.
drop table if exists declaration_send;
drop table if exists assignment_generation;
drop table if exists build_gate;
drop table if exists duration;
internal/inventory/nodes.go
+5 -35
View File
@@ -37,8 +37,6 @@ type Inventory struct {
// chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain).
chain *chain.Chain
state *chainState
// recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store.
recorded func(context.Context, Duration) error
// throwaway is set on an inventory raised for a moment beside the real mesh (the merge gate's): nothing it
// composes is to be written to the store or the bus.
throwaway bool
@@ -73,7 +71,6 @@ func Open(ctx context.Context) (*Inventory, error) {
}
func (i *Inventory) Close() {
i.stopViews()
i.store.Close()
}
@@ -1283,8 +1280,9 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
`update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil {
return err
}
// What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record.
return i.recordRuns(ctx, node, builds, time.Now())
// What it was sent of each module is no longer kept in the assignments: they hold placement only (ADR 0539 §12).
_ = builds
return nil
}
// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue
@@ -1301,36 +1299,8 @@ func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time,
// assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent
// anything, or last sent a declaration by hand, which makes every key of it say its commit unknown.
func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) {
var sentAt *time.Time
err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
if err != nil || sentAt == nil {
return nil, false, err
}
c, err := i.theChain(ctx)
if err != nil {
return nil, false, err
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err)
}
builds = map[string]string{}
for _, k := range all {
a := k.Value
if a.Node != name || !a.Stands() {
continue
}
if !a.SentUnknown.IsZero() {
return nil, false, nil
}
if a.Commit != "" {
builds[a.Module] = a.Commit
}
}
return builds, true, nil
// Never known: the assignments hold placement only (ADR 0539 §12), and what a machine runs is its have.
return nil, false, nil
}
// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
internal/inventory/older_build_test.go
-126
View File
@@ -1,126 +0,0 @@
//go:build integration
package inventory
import (
"context"
"errors"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// novox/hq 04-ISSUES/219: two builds of one module in flight together, the one asked first heard
// last. The newer request stood on the newer base; the older one's late outcome is recorded and is
// not what the module is.
func postgresBuild(id string, asked time.Time, image string) Build {
b := aBuild(id, "postgres", "")
b.Asked = asked
b.Against = []string{"mesh-tools/runtime@sha256:" + id}
b.Made = []Artifact{{Name: "server", Kind: "image", Reference: "postgres@sha256:" + image}}
return b
}
func TestAnOlderRequestFinishingLaterIsNotWhatTheModuleHolds(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
// The newer request finishes first, the older one last — recorded in that order.
if err := inv.RecordBuild(ctx, postgresBuild("newer", newer, "4bcd5f73")); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, postgresBuild("older", older, "0ab07fa9")); err != nil {
t.Fatal(err)
}
held, err := inv.Held(ctx)
if err != nil {
t.Fatal(err)
}
if got := held["postgres/server"]; got != "postgres@sha256:4bcd5f73" {
t.Errorf("postgres holds %q; want the newer request's image 4bcd5f73", got)
}
against, err := inv.BuiltAgainst(ctx)
if err != nil {
t.Fatal(err)
}
if got := against["postgres"]; len(got) != 1 || got[0] != "mesh-tools/runtime@sha256:newer" {
t.Errorf("postgres stands on %v; want what the newer request stood on", got)
}
// Both are still recorded, the late one first as what happened lately.
builds, err := inv.Builds(ctx, "postgres", 5)
if err != nil {
t.Fatal(err)
}
if len(builds) != 2 || builds[0].ID != "older" || !builds[0].Asked.Equal(older) {
t.Fatalf("both builds, newest heard first, with when they were asked: %+v", builds)
}
}
func TestABuildWithNoKnownRequestTimeIsOrderedByWhenItWasRecorded(t *testing.T) {
// What the mesh did before it kept the request time, so a row from before still answers.
inv := fresh(t)
ctx := context.Background()
for _, id := range []string{"first", "second"} {
b := aBuild(id, "shell", "")
b.Made = []Artifact{{Name: "config", Kind: "archive", Reference: "…/" + id}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
held, err := inv.Held(ctx)
if err != nil {
t.Fatal(err)
}
if got := held["shell/config"]; got != "…/second" {
t.Errorf("shell holds %q; want the one recorded last", got)
}
}
func TestARegistrationFromAnOlderRequestDoesNotReplaceANewerOne(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
from := func(asked time.Time) Source {
return Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/postgres",
BuiltFrom: "efff5415", Asked: asked}
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "fixed"}, from(newer)); err != nil {
t.Fatal(err)
}
err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "stale"}, from(older))
if !errors.Is(err, ErrSuperseded) {
t.Fatalf("an older request's registration was not refused as superseded: %v", err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
if got := shelf["postgres"].Version; got != "fixed" {
t.Fatalf("postgres is %q; want the newer request's manifest", got)
}
// A later request, and a manifest handed over by hand — asked when it is handed over — both
// replace it as before.
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "later"},
from(newer.Add(time.Minute))); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "by-hand"}, Source{}); err != nil {
t.Fatal(err)
}
if shelf, _ := inv.Catalogue(ctx); shelf["postgres"].Version != "by-hand" {
t.Fatalf("postgres is %q; want the manifest handed over by hand", shelf["postgres"].Version)
}
src, err := inv.SourceOf(ctx, "postgres")
if err != nil || src.Repository != "novox/mesh-catalog" {
t.Fatalf("a hand registration erased the provenance: %+v %v", src, err)
}
}
internal/inventory/order_test.go
-128
View File
@@ -1,128 +0,0 @@
//go:build integration
package inventory
import (
"context"
"errors"
"testing"
"time"
)
// Order and one writer, as the store keeps them (novox/hq to-be 45 §6).
// **A plan is written by compare-and-set on its revision, carrying the epoch**: a write against a plan
// another writer moved since is refused, and nothing is written by a process that may not act.
func TestAPlanIsWrittenByCompareAndSetUnderTheLease(t *testing.T) {
inv := ForTest(t)
ctx := context.Background()
epoch := uint64(57)
inv.ActsUnder(func(context.Context) (uint64, error) { return epoch, nil })
p := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: PlanBuilding,
Tiers: [][]string{{"app"}}, Modules: map[string]*PlanModule{"app": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
if p.Revision != 1 || p.Epoch != 57 {
t.Fatalf("a new plan was written at revision %d, epoch %d", p.Revision, p.Epoch)
}
// Two readers of revision 1: the first write wins, the second is refused and writes nothing.
first, err := inv.PlanByID(ctx, "plan-cas")
if err != nil {
t.Fatal(err)
}
second := first
first.Note = "the newer word"
if err := inv.SavePlan(ctx, &first); err != nil {
t.Fatal(err)
}
second.State = PlanFailed
if err := inv.SavePlan(ctx, &second); !errors.Is(err, ErrPlanMoved) {
t.Fatalf("a write against a plan moved since it was read was not refused: %v", err)
}
kept, _ := inv.PlanByID(ctx, "plan-cas")
if kept.State != PlanBuilding || kept.Note != "the newer word" || kept.Revision != 2 {
t.Fatalf("the refused write reached the plan: %+v", kept)
}
// The writer saves its own plan again without reading it: its revision moved with its write.
first.Tier = 0
if err := inv.SavePlan(ctx, &first); err != nil {
t.Fatalf("a writer could not save its own plan twice: %v", err)
}
// A new plan under an id already written is refused, not laid over it.
again := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "deadbeef", Created: time.Now(), State: PlanBuilding}
if err := inv.SavePlan(ctx, &again); !errors.Is(err, ErrPlanMoved) {
t.Fatalf("a second plan under one id was written: %v", err)
}
// And a process that does not hold the lease writes nothing.
inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") })
first.Note = "from a controller that lost the lease"
if err := inv.SavePlan(ctx, &first); err == nil {
t.Fatal("a plan was written by a controller that does not hold the lease")
}
if kept, _ := inv.PlanByID(ctx, "plan-cas"); kept.Note != "the newer word" || kept.Epoch != 57 {
t.Fatalf("a controller without the lease wrote the plan: %+v", kept)
}
}
// The epochs the mesh issued: the highest is the floor; taking one ends every earlier one nobody gave
// back as found expired, and says which; one given back says so and is not said again.
func TestTheEpochsIssuedAreKept(t *testing.T) {
inv := ForTest(t)
ctx := context.Background()
if highest, err := inv.HighestEpoch(ctx); err != nil || highest != 0 {
t.Fatalf("a mesh that issued none has %d (%v)", highest, err)
}
if _, err := inv.TookEpoch(ctx, Epoch{Epoch: 41, Instance: "a", Taken: time.Now()}); err != nil {
t.Fatal(err)
}
// a stopped renewing and said nothing; b takes the lease.
expired, err := inv.TookEpoch(ctx, Epoch{Epoch: 57, Instance: "b", Taken: time.Now()})
if err != nil || len(expired) != 1 || expired[0].Epoch != 41 || expired[0].How != EpochExpired {
t.Fatalf("taking 57 ended %+v (%v), want 41 found expired", expired, err)
}
if err := inv.EndEpoch(ctx, 57, EpochReleased); err != nil {
t.Fatal(err)
}
expired, err = inv.TookEpoch(ctx, Epoch{Epoch: 60, Instance: "c", Taken: time.Now()})
if err != nil || len(expired) != 0 {
t.Fatalf("a lease given back was found expired: %+v (%v)", expired, err)
}
if highest, _ := inv.HighestEpoch(ctx); highest != 60 {
t.Fatalf("the highest epoch issued is %d, want 60", highest)
}
e, found, err := inv.EpochOf(ctx, 57)
if err != nil || !found || e.Instance != "b" || e.How != EpochReleased || e.Ended == nil {
t.Fatalf("epoch 57 reads %+v (%v, %v)", e, found, err)
}
recent, err := inv.EpochsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(recent) != 3 || recent[0].Epoch != 60 || recent[0].Ended != nil {
t.Fatalf("the epochs of the last hour read %+v (%v)", recent, err)
}
}
// What a machine was sent under, and whether it reads an epoch.
func TestTheEpochAMachineWasSentIsKept(t *testing.T) {
inv := ForTest(t)
ctx := context.Background()
node, err := inv.AddNode(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSentUnder(ctx, node.ID, "d1", nil, 57); err != nil {
t.Fatal(err)
}
if e, err := inv.SentEpoch(ctx, node.ID); err != nil || e != 57 {
t.Fatalf("sent under %d (%v)", e, err)
}
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
t.Fatal(err)
}
if e, _ := inv.SentEpoch(ctx, node.ID); e != 0 {
t.Fatalf("a declaration sent without an epoch left %d kept", e)
}
if reads, _ := inv.ReadsEpoch(ctx, node.ID); reads {
t.Fatal("a machine that never said so reads an epoch")
}
}
internal/inventory/plan_backups_test.go
-35
View File
@@ -1,35 +0,0 @@
package inventory
import (
"encoding/json"
"reflect"
"strings"
"testing"
"time"
)
// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names
// are fixed here, and a plan keeps them through a round trip.
func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) {
at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"},
Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}}
raw, err := json.Marshal(p)
if err != nil {
t.Fatal(err)
}
for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` +
`"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} {
if !strings.Contains(string(raw), name) {
t.Fatalf("plan JSON %s lacks %s", raw, name)
}
}
var back Plan
if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) {
t.Fatalf("round trip: %v %+v", err, back.Backups)
}
raw, _ = json.Marshal(Plan{ID: "q"})
if strings.Contains(string(raw), "backups") {
t.Fatalf("a plan with no backups says none: %s", raw)
}
}
internal/inventory/plan_id_test.go
-47
View File
@@ -1,47 +0,0 @@
//go:build integration
package inventory
import (
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// **A plan's id is a value, never a query** (novox/hq issue 561): the id reaches PlanByID from a caller of the
// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no
// other plan answered for it. On the bus an id is a key, and one the bus takes no spelling of is refused to be kept.
func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(),
State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
for _, id := range []string{
"x' or '1'='1", // every plan, when pasted into the query
"plan-561' or id like '%", // the plan above, by a pattern
"it's", // a quote alone: a syntax error, when pasted
} {
got, err := inv.PlanByID(ctx, id)
if err == nil {
t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID)
}
if want := "no plan " + id; err.Error() != want {
t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want)
}
}
// A plan whose id holds a quote cannot be a key of the bus: refused by name, never rewritten into another id.
quoted := p
quoted.ID, quoted.Revision = "plan-'561'", 0
if err := inv.SavePlan(ctx, &quoted); !errors.Is(err, chain.ErrBadKey) {
t.Fatalf("a plan with a quote in its id was kept, or refused for another reason: %v", err)
}
if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") {
t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent)
}
}
internal/inventory/plans.go
-765
View File
@@ -1,765 +0,0 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
// still waits for.
type Plan struct {
ID string `json:"id"`
Repository string `json:"repository"`
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
// ones of the same repository and branch. Empty for a plan from before it was kept.
Branch string `json:"branch,omitempty"`
Commit string `json:"commit"`
// Merged is when the forge made the merge this plan answers (novox/hq issue 349): the order of a
// branch's merges, which is not the order their plans were made in when one was acted on late. Zero
// for a release, and for a plan kept before it was.
Merged time.Time `json:"merged,omitzero"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
State string `json:"state"`
Tier int `json:"tier"`
Tiers [][]string `json:"tiers"`
Modules map[string]*PlanModule `json:"modules"`
Note string `json:"note,omitempty"`
// Away is the machines the walk passed because they were out of touch as a tier was sent (novox/hq ADR 0435 §1,
// issue 1161): not sent to and not waited for. Their want of the walk's builds stays for the send they get when
// they are heard again (ADR 0360 §5). Sorted.
Away []string `json:"away,omitempty"`
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
// never written from here: a save measures the tier it leaves and stamps the next. What the
// watchdog of a plan's progress (S3) reads.
TierEntered time.Time `json:"tier_entered,omitempty"`
// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
// plan is written by compare-and-set, so a write against a plan another writer moved since is
// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
Revision int64 `json:"revision"`
// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
Epoch uint64 `json:"epoch,omitempty"`
// Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a
// gate, walked through the machines one at a time.
Release *PlanRelease `json:"release,omitempty"`
// Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239), and on every walk and
// batch that names the merges it answers (ADR 0276): Awaits empty for a walk on the controller's own path,
// which starts when it is cut.
Delivery *PlanDelivery `json:"delivery,omitempty"`
// Commits is every repository's commit the walk carries, one per repository (novox/hq ADR 0276): a batch's
// walk can carry several. Repository and Commit above keep one of them, for a reader that knows one. Empty
// for a walk kept before it was: Repository and Commit are then the whole of it.
Commits []PlanCommit `json:"commits,omitempty"`
// Times are the moments of a walk no other field keeps (novox/hq ADR 0282 decision 6): when its batch's
// window closed, when it was cut, and its class. Nil for a walk kept before they were, whose phases before
// its build are said unknown.
Times *PlanTimes `json:"times,omitempty"`
// Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out
// from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`).
Phases *WalkPhases `json:"phases,omitempty"`
// Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3):
// per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a
// walk that moves no module with an item protected by a backup, and for a walk kept before it was.
Backups *PlanBackups `json:"backups,omitempty"`
// Links are what the rollout names of the chain (novox/hq ADR 0408 §1): the builds it carried, the machine steps it has
// taken and the assignments it changed, each with the build before and after. Kept in the rollout's record beside
// the fields above; empty in the controller's store.
chain.Links
// Held are the merges this plan or batch holds (novox/hq ADR 0276, ADR 0408): what the store kept as batched merges
// with this plan's id, and the merges no plan answers that it held last. **Written by the merge operations only**: a
// save of the plan keeps what the record holds, never what this copy read, so a copy read before a merge moved cannot
// lay it back (issue 1092).
Held []chain.HeldMerge `json:"held,omitempty"`
// WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308
// decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every
// other save.
WalkSaid string `json:"-"`
}
// The states of a plan's backups, as a whole and each.
const (
BackupPending = "pending"
BackupDone = "done"
BackupFailed = "failed"
)
// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by
// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories.
type PlanBackups struct {
// State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one
// failed, done when every one is.
State string `json:"state"`
// Asked is one entry per machine and module, sorted by machine then module.
Asked []PlanBackup `json:"asked"`
// NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup:
// nothing is asked of them, and the plan says so.
NoHolder []string `json:"no_holder,omitempty"`
}
// PlanBackup is one backup asked of one machine's backup holder for one module.
type PlanBackup struct {
Machine string `json:"machine"`
Module string `json:"module"`
// State is pending, done or failed.
State string `json:"state"`
// AskedAt is when the holder was asked; nil while it is not asked yet.
AskedAt *time.Time `json:"asked_at,omitempty"`
// TriedAt is when the controller first tried to ask, kept while the holder cannot be reached: the bound of a
// backup that is never asked counts from it. Nil once asked.
TriedAt *time.Time `json:"tried_at,omitempty"`
// Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one.
Words string `json:"words,omitempty"`
}
// PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6).
type PlanTimes struct {
// WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum.
// Nil for a walk no window assembled (one merge walked alone).
WindowClosed *time.Time `json:"window_closed,omitempty"`
// Cut is when the batch became the walk.
Cut *time.Time `json:"cut,omitempty"`
// Class is the walk's module class, read at its cut: core when it moves a module on the controller's own
// path or the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
Class string `json:"class,omitempty"`
// Started is when the walk was said started, and Ended when it was said ended (novox/hq ADR 0308 decision
// 1): each said once, on the save that marked it.
Started *time.Time `json:"started,omitempty"`
Ended *time.Time `json:"ended,omitempty"`
}
// PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch.
type PlanCommit struct {
Repository string `json:"repository"`
Branch string `json:"branch,omitempty"`
Commit string `json:"commit"`
Merged time.Time `json:"merged,omitzero"`
}
// PlanMerge is one merge a walk or batch answers (novox/hq ADR 0276), as mesh-delivery reads it: Carried is
// the commit of its repository the walk builds and sends when that is a later merge containing it, and
// empty when it is the merge itself.
type PlanMerge struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Carried string `json:"carried,omitempty"`
// Number and Title are the merge's pull request as the forge announced it, and Moves the modules the merge
// moved when it was heard: what `plans` names a walk by. Empty where the announcement said none.
Number int `json:"number,omitempty"`
Title string `json:"title,omitempty"`
Moves []string `json:"moves,omitempty"`
// Head is the pull request's head commit the forge merged (novox/hq issue 703): the walk takes only a build a
// check of that head made. Empty where the announcement said none, and then it takes none.
Head string `json:"head,omitempty"`
// Merged is when the forge made the merge and Heard when the controller heard it (novox/hq ADR 0282): where
// its delivery time starts. Zero for a merge named before they were kept.
Merged time.Time `json:"merged,omitzero"`
Heard time.Time `json:"heard,omitzero"`
}
// PlanBatch is a batch's window while it is one (novox/hq ADR 0276): when it closes unless another merge
// comes, when it closes at the latest, and the walk it waits behind once closed.
type PlanBatch struct {
ClosesAt time.Time `json:"closes_at"`
AtMost time.Time `json:"at_most"`
Behind string `json:"behind,omitempty"`
// BehindWhy says why the batch waits behind that walk: what the two share (issue 973).
BehindWhy string `json:"behind_why,omitempty"`
// Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
// build, 2026-10-10).
Own bool `json:"own,omitempty"`
// Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they
// were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are
// the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so
// the batch's walk does not build them.
Closed bool `json:"closed,omitempty"`
Withheld []string `json:"withheld,omitempty"`
// CutAtOnce says the batch's window closed when it was made: its merge shares nothing with a batch
// assembling and has no delivery group member still to merge, so it waits for no window (novox/hq ADR 0304
// decision 1, ADR 0306 decisions 2 and 3). A merge that joins it later opens its window again. It is not
// Closed, which is a stop's.
CutAtOnce bool `json:"cut_at_once,omitempty"`
// Waits says why a batch made for one merge waits for the window: its delivery group has a member still to
// merge, or mesh-delivery could not say whether it has (novox/hq ADR 0306 decision 2).
Waits string `json:"waits,omitempty"`
// Holds are the groups whose last member is still to merge, by head branch name, each with why in words
// (novox/hq ADR 0418 clause 3): the batch does not close by the merge window while one is held, however long the
// member takes. Cleared when every member is merged or closed; the window then runs again from that moment.
Holds map[string]string `json:"holds,omitempty"`
// HeldAt is when the last hold was cleared: the maximum is counted from it, not from the batch's making.
HeldAt time.Time `json:"held_at,omitempty"`
}
// OwnPath says the record is a batch of merges on the controller's own path.
func (p Plan) OwnPath() bool {
return p.Delivery != nil && p.Delivery.Batch != nil && p.Delivery.Batch.Own
}
// CommitOf is the commit of a repository the walk carries; empty when it carries none of it.
func (p Plan) CommitOf(repository string) string {
for _, c := range p.Commits {
if strings.EqualFold(c.Repository, repository) {
return c.Commit
}
}
if strings.EqualFold(p.Repository, repository) {
return p.Commit
}
return ""
}
// CommitOn is the commit of a repository's branch the walk carries; empty when it carries none of it. A
// commit kept without its branch matches any branch.
func (p Plan) CommitOn(repository, branch string) string {
for _, c := range p.Commits {
if strings.EqualFold(c.Repository, repository) && (branch == "" || c.Branch == "" || c.Branch == branch) {
return c.Commit
}
}
if len(p.Commits) == 0 && strings.EqualFold(p.Repository, repository) &&
(branch == "" || p.Branch == "" || p.Branch == branch) {
return p.Commit
}
return ""
}
// Carried is every repository's commit the walk carries: Commits, or its one repository and commit.
func (p Plan) Carried() []PlanCommit {
if len(p.Commits) > 0 {
return p.Commits
}
if p.Repository == "" && p.Commit == "" {
return nil
}
return []PlanCommit{{Repository: p.Repository, Branch: p.Branch, Commit: p.Commit, Merged: p.Merged}}
}
// Named is the walk as a person reads it: each repository at its commit.
func (p Plan) Named() string {
var out []string
for _, c := range p.Carried() {
out = append(out, c.Repository+" "+short(c.Commit))
}
return strings.Join(out, ", ")
}
// Batch says the record is a batch still being assembled or waiting to be cut, not a walk (ADR 0276).
func (p Plan) Batch() bool { return p.State == PlanAssembling || p.State == PlanQueued }
// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239).
type PlanDelivery struct {
// Order are the edges the plan's own order was drawn with (novox/hq ADR 0419): verb before caller, version skew,
// engine before controller. Kept on the plan because the catalogue's graph does not hold them, and the plan's gates
// (a later tier waits for the earlier to be applied) are read from them.
Order []Edge `json:"order,omitempty"`
// Awaits is who must say the walk may start: the seat whose holder owns the delivery.
Awaits string `json:"awaits"`
// Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why.
Go *time.Time `json:"go,omitempty"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
// Stopped is who ended the walk, through the delivery's owner or by hand (`plans stop|close`), and StoppedWhy
// why: the walk is failed, and the delivery reads it as stopped, not as a build that failed.
Stopped string `json:"stopped,omitempty"`
StoppedWhy string `json:"stopped_why,omitempty"`
// Merges is every merge the walk or batch answers (novox/hq ADR 0276): its own commits, and the earlier
// merges of a repository its later commit contains.
Merges []PlanMerge `json:"merges,omitempty"`
// TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that
// walk's now.
TakenOverBy string `json:"taken_over_by,omitempty"`
// WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges
// again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out
// ends stopped with the walk, one of a merge walked again follows that batch's walk.
WalkedAgainBy string `json:"walked_again_by,omitempty"`
LeftOut []PlanLeftOut `json:"left_out,omitempty"`
// Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running
// the stopped walk's build (novox/hq ADR 0299).
Kept []PlanKept `json:"kept,omitempty"`
// Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a
// stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299).
Withheld []string `json:"withheld,omitempty"`
// Unchanged are the modules a merge it answers changed files in the directory of, whose recorded build source
// holds none of them, each with why (novox/hq issue 611, ADR 0267 §5): not built, because the build the mesh
// holds is their build. A delivery plan that read them whole named them; this says they matched what runs,
// rather than that the walk left them unbuilt. Modules keeps meaning what the walk builds.
Unchanged map[string]string `json:"unchanged,omitempty"`
// Batch is the window of a batch; nil once it is cut into a walk.
Batch *PlanBatch `json:"batch,omitempty"`
// Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one:
// built at that commit, not at the branch.
Alone bool `json:"alone,omitempty"`
}
// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR
// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so
// cannot be walked without delivering it.
type PlanLeftOut struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge
// stopped itself.
Contains string `json:"contains,omitempty"`
// Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped
// merge, they would deliver it.
Shares []string `json:"shares,omitempty"`
}
// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached.
type PlanKept struct {
Module string `json:"module"`
Machines []string `json:"machines"`
}
// Waiting is whether the walk waits for its delivery's word.
func (p Plan) Waiting() bool {
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
}
// PlanSaved is told every plan this process kept, after it is kept, with the inventory that kept it (novox/hq
// ADR 0239): the serving controller says it on the bus as `plan-moved`, and so does a command (the controller's
// run installs it for every verb, novox/hq ADR 0308 decision 1), so the save that starts or ends a walk is said
// whichever process made it. Nil says nothing; whoever follows a walk also asks for it, and finds an unsaid save
// by comparison.
var PlanSaved func(*Inventory, Plan)
// ErrPlanMoved is a save against a plan written by somebody else since it was read.
var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
// PlanModule is one module's state within a plan.
type PlanModule struct {
// State: asked, built, failed; empty for a module whose tier has not been asked yet.
State string `json:"state,omitempty"`
AskedAt *time.Time `json:"asked_at,omitempty"`
BuiltAt *time.Time `json:"built_at,omitempty"`
// BusWaitFrom is when the bus first did not answer an ask of this module (issue 1334): the module stays unasked and is
// asked again at the next look, until the bus has been silent for the bound, when the plan fails as for any refusal.
BusWaitFrom *time.Time `json:"bus_wait_from,omitempty"`
// SentAt is when the plan sent the machines running this module its new build, because a
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
// after this.
SentAt *time.Time `json:"sent_at,omitempty"`
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
// replaced while the plan waits on that report resumes the wait rather than sending again. The
// machine holding the bus is among them when its user list had to go first.
First []string `json:"first,omitempty"`
FirstAt *time.Time `json:"first_at,omitempty"`
Commit string `json:"commit,omitempty"`
Why string `json:"why,omitempty"`
// Placed is the edge that put the module after another, and the rule that drew it (novox/hq ADR 0419 clause 5):
// "after <module> (verb before caller: <seat>.<verb>)". Empty for a module placed first by nothing.
Placed string `json:"placed,omitempty"`
// Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>", naming the pull
// request's head, the trunk commit and the digests, or "built: <why>". Said in `plan-moved` and kept here.
Made string `json:"made,omitempty"`
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
// its module's name included. Empty in a plan from before it was kept, which is matched by
// module, or by repository and path, as before.
Build string `json:"build,omitempty"`
// Previous is the build the first machine ran of this module before the plan sent it the new one —
// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
// (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent
// is not known.
Previous string `json:"previous,omitempty"`
// Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a
// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
Gate *PlanGate `json:"gate,omitempty"`
// GatedBy names the module of the same tier whose gate judges this one on its first machine: they
// went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for
// the module the gate is kept on, and for a plan from before tiers were sent whole.
GatedBy string `json:"gated_by,omitempty"`
// Rest is, per machine of the rest, the declaration the send after the first-node gate carried there (novox/hq ADR
// 0282 decision 6): the machine's first report of it, applied, is when the build runs there.
Rest map[string]SentDeclaration `json:"rest,omitempty"`
// Recorded is, for a module whose policy records, per machine running it with a move, which way the walk took it
// (novox/hq ADR 0377 §6): "went out: …" when the move recreates none of its containers and moves none of its
// data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it
// could not be judged.
Recorded map[string]string `json:"recorded,omitempty"`
// Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453
// §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome.
Key bool `json:"key,omitempty"`
// KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty
// for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now.
KeyAt string `json:"key_at,omitempty"`
// Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value;
// empty until the indexer has written it.
Fingerprint string `json:"fingerprint,omitempty"`
}
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
type PlanGate struct {
// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
// for any other module, judged by its own health.
Component string `json:"component,omitempty"`
Machines []string `json:"machines"`
From string `json:"from,omitempty"`
To string `json:"to,omitempty"`
// Since is when the judging began: the first machine reported the new build applied.
Since *time.Time `json:"since,omitempty"`
// Sent is, per machine, the declaration the gate's send carried there (novox/hq issue 352): what a
// machine's report is held against. Absent on a gate kept before it was, which reads the report
// against the send made last, as before.
Sent map[string]SentDeclaration `json:"sent,omitempty"`
// Applied is, per machine, the time of the first report the gate read of its send applied: the
// node-engine started the build before it, so a witness window counted from it ends no earlier than the
// node-engine's own (novox/hq ADR 0351 decisions 1 and 4).
Applied map[string]time.Time `json:"applied,omitempty"`
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
// does not resets them.
Passes int `json:"passes,omitempty"`
// Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two
// minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR
// 0282 decision 4 and ADR 0304 decision 3). Zero in a gate
// judged before it was said: three.
Wants int `json:"wants,omitempty"`
LastPass *time.Time `json:"last_pass,omitempty"`
// Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading
// rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352).
Faulted bool `json:"faulted,omitempty"`
// HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor
// counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3).
HealthySeen bool `json:"healthy_seen,omitempty"`
// Last is what the newest judging found wanting, while it still may pass.
Last string `json:"last,omitempty"`
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
Verdict string `json:"verdict,omitempty"`
Why string `json:"why,omitempty"`
JudgedAt *time.Time `json:"judged_at,omitempty"`
Took string `json:"took,omitempty"`
// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
Rollback string `json:"rollback,omitempty"`
// Kept says a passing verdict was written to the gate's records.
Kept bool `json:"kept,omitempty"`
// Carried is every module whose build moved on the judged machines with the send — the plan's own
// module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a
// pass is its verdict too, and one that fails is put back.
Carried []CarriedMove `json:"carried,omitempty"`
// Failing names the modules the last judging found wanting.
Failing []string `json:"failing,omitempty"`
// Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged,
// or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's.
Healthy map[string]int `json:"healthy,omitempty"`
// Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never
// a reason to fail.
Waits map[string]string `json:"waits,omitempty"`
// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
// pass, and is not put back.
Passing []string `json:"passing,omitempty"`
// HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery
// after the one before (issue 318 review).
HealthyAt map[string]time.Time `json:"healthy_at,omitempty"`
// Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the
// first reason: the send fails, and the modules beside them are judged to their own verdict first.
Broken []string `json:"broken,omitempty"`
BrokenWhy string `json:"broken_why,omitempty"`
// Returned names the broken modules already put back, at once, while the rest of the send is judged.
Returned []string `json:"returned,omitempty"`
// Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that
// counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept.
Readings []GateReading `json:"readings,omitempty"`
// Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent
// alone to the machine holding the bus, every build there kept, so what the send newly grants is on the
// bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus.
Grants *GrantsStep `json:"grants,omitempty"`
// Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine
// reports its module's container left running because it said busy, the judging waits and has not
// started; once it reports the container replaced, Since is moved to then. Nil when nothing waited.
Busy *GateBusy `json:"busy,omitempty"`
// Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that
// takes over reads the machine's last send before making it, and never sends it twice.
Pending bool `json:"pending,omitempty"`
// Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left,
// neither failed nor put back.
Silent bool `json:"silent,omitempty"`
// TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of
// the same build on the same machine, which this gate goes on from instead of starting from zero.
TookOver string `json:"took_over,omitempty"`
}
// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR
// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found.
// A module's own count is taken only for the modules this gate judges.
func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) {
if from == nil {
return
}
if from.Since != nil {
since := *from.Since
g.Since = &since
}
for n, s := range from.Sent {
if slices.Contains(g.Machines, n) {
if g.Sent == nil {
g.Sent = map[string]SentDeclaration{}
}
g.Sent[n] = s
}
}
for n, at := range from.Applied {
if slices.Contains(g.Machines, n) {
if g.Applied == nil {
g.Applied = map[string]time.Time{}
}
g.Applied[n] = at
}
}
g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last
if from.LastPass != nil {
at := *from.LastPass
g.LastPass = &at
}
for _, m := range modules {
if n, counted := from.Healthy[m]; counted {
if g.Healthy == nil {
g.Healthy = map[string]int{}
}
g.Healthy[m] = n
}
if at, counted := from.HealthyAt[m]; counted {
if g.HealthyAt == nil {
g.HealthyAt = map[string]time.Time{}
}
g.HealthyAt[m] = at
}
}
g.Readings = append([]GateReading(nil), from.Readings...)
g.TookOver = id
}
// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4).
type GateBusy struct {
Machine string `json:"machine"`
Module string `json:"module"`
Container string `json:"container"`
// Since is when the machine's first apply that would have replaced it ran, Until when its node-engine
// replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when
// the controller stopped waiting past the engine's bound, which PastBound says.
Since time.Time `json:"since"`
Until time.Time `json:"until"`
// Seen is when this controller first read the wait, on its own clock: the gate's bound runs from it, never
// from the engine's word alone.
Seen time.Time `json:"seen,omitzero"`
Ended *time.Time `json:"ended,omitempty"`
PastBound bool `json:"past_bound,omitempty"`
// Said is the last output of its busy command, as the machine reported it.
Said string `json:"said,omitempty"`
}
// Waiting says the gate waits for a busy container and has not started.
func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil }
// Words is the wait in a line, as the walk says it.
func (b *GateBusy) Words() string {
said := ""
if b.Said != "" {
said = "; its busy command says: " + b.Said
}
return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+
"at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said)
}
// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq
// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it.
type GrantsStep struct {
Node string `json:"node"`
At *time.Time `json:"at,omitempty"`
Failed string `json:"failed,omitempty"`
}
// GateReading is one reading of a first-node gate.
type GateReading struct {
At time.Time `json:"at"`
Healthy bool `json:"healthy"`
// Said is what a reading that did not pass found wanting.
Said string `json:"said,omitempty"`
}
// maxReadings bounds a first-node gate's readings: a judging that never passes reads every few seconds for ten minutes.
const maxReadings = 24
// Read keeps one reading: a pass always, and a reading that did not pass only when the one before passed or
// there is none, so a judging waiting for a module to start keeps one line of it, not hundreds.
func (g *PlanGate) Read(at time.Time, healthy bool, said string) {
if !healthy && len(g.Readings) > 0 && !g.Readings[len(g.Readings)-1].Healthy {
return
}
if r := []rune(said); len(r) > 200 {
said = string(r[:200])
}
g.Readings = append(g.Readings, GateReading{At: at, Healthy: healthy, Said: said})
if len(g.Readings) > maxReadings {
g.Readings = g.Readings[len(g.Readings)-maxReadings:]
}
}
// CarriedMove is one module's build moving on a machine with a gated send.
type CarriedMove struct {
Module string `json:"module"`
Node string `json:"node"`
From string `json:"from,omitempty"`
To string `json:"to"`
Build string `json:"build,omitempty"`
// Recreates is what the send does to the module's containers, in the mesh's words — how many it
// recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when
// it recreates none, or when the build the machine ran is not known.
Recreates string `json:"recreates,omitempty"`
}
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
// that waits for a gate, sent one machine at a time, each judged before the next.
type PlanRelease struct {
Order []string `json:"order"`
Next int `json:"next"`
// Gate is the first machine being judged; nil between machines.
Gate *PlanGate `json:"gate,omitempty"`
// Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by
// its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2).
Rest bool `json:"rest,omitempty"`
Gates map[string]*PlanGate `json:"gates,omitempty"`
// Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send
// within ApplySilentAfter (ADR 0353 §3 and §4).
Failed []string `json:"failed,omitempty"`
Silent []string `json:"silent,omitempty"`
Done []string `json:"done,omitempty"`
// Skipped are the machines not heard from when their turn came, left as they were.
Skipped []string `json:"skipped,omitempty"`
// By is the person who released it, empty when the mesh did.
By string `json:"by,omitempty"`
// Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings
// layer the operator approved, sent to the machines it reaches one at a time, its module judged on each.
Settings *PlanSettings `json:"settings,omitempty"`
}
// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the
// layer's scope, the ask that approved it and what changed, in words.
type PlanSettings struct {
Module string `json:"module"`
// Node is the machine the layer is for, empty for the whole mesh.
Node string `json:"node,omitempty"`
// Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs").
Ask string `json:"ask"`
Change string `json:"change,omitempty"`
}
// The states a plan passes through.
const (
PlanBuilding = "building"
PlanRolling = "rolling"
PlanDone = "done"
PlanFailed = "failed"
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
PlanSuperseded = "superseded"
// PlanAssembling is a batch whose merge window is open (novox/hq ADR 0276), and PlanQueued one whose
// window closed while a walk is open: neither is a walk yet. Cut, a batch keeps its id and is building.
PlanAssembling = "assembling"
PlanQueued = "queued"
)
// Open says whether the plan is still being worked.
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
//
// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
// On success p's revision and epoch are the ones written, so the caller may save it again.
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
return i.SavePlansMoving(ctx, []*Plan{p}, nil)
}
// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276).
type MergeMove struct {
Repository, Commit, Plan string
Alone bool
// Event replaces the merge's kept announcement when it is put back to waiting (Plan empty); nil leaves it.
Event json.RawMessage
}
// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one
// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
// the transaction is committed, in order.
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
c, err := i.theChain(ctx)
if err != nil {
return err
}
return i.chainSavePlans(ctx, c, plans, moves)
}
// short is a commit as a person reads it.
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
// OpenPlans is every plan still being worked, oldest first.
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
}
// PlansSince is every plan made after a moment, and every plan still being worked, oldest first.
func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
}
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
if err == nil && limit >= 0 && len(plans) > limit {
plans = plans[:limit]
}
return plans, err
}
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
// recorded when its merge was made.
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
c, err := i.theChain(ctx)
if err != nil {
return Plan{}, false, err
}
return i.chainNewestMergeOf(ctx, c, repository, branch)
}
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
// caller of the plans verb.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
c, err := i.theChain(ctx)
if err != nil {
return Plan{}, err
}
return i.chainPlanByID(ctx, c, id)
}
internal/inventory/plans_leave_the_store_test.go
+47
View File
@@ -0,0 +1,47 @@
//go:build integration
package inventory
import (
"strings"
"testing"
)
// **Migration 0108 drops the tables of the delivery plans' machinery** (novox/hq ADR 0539 §17, design 47 release 6, issue 1945):
// the record of each send, the assignment generation, the gate's verdicts and the measured durations. The store is built as
// the mesh holds it before the migration, with a row in each, and after it the four are gone and what the controller still
// keeps stays: the node and its columns, the module, the bus step, the core box and the refused registrations.
func TestMigration0108DropsTheTablesOfThePlansMachineryAndKeepsWhatTheControllerStillKeeps(t *testing.T) {
inv, migrate := ForTestBefore(t, 108)
tables := func() map[string]bool {
t.Helper()
out := map[string]bool{}
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
out[strings.Trim(row[0], `"`)] = true
}
return out
}
gone := []string{"declaration_send", "assignment_generation", "build_gate", "duration"}
before := tables()
for _, name := range gone {
if !before[name] {
t.Fatalf("the table %s is not in the store before the migration", name)
}
}
migrate()
after := tables()
for _, name := range gone {
if after[name] {
t.Errorf("the table %s is still in the store", name)
}
}
for _, stays := range []string{"node", "module", "bus_step", "core_box", "build_refused", "controller_schema"} {
if !after[stays] {
t.Errorf("the table %s is gone", stays)
}
}
// The functions and triggers the tables carried go with them.
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
t.Errorf("the function raise_assignment_generation is still in the store")
}
}
internal/inventory/plans_test.go
-225
View File
@@ -1,225 +0,0 @@
//go:build integration
package inventory
import (
"reflect"
"testing"
"time"
)
// A plan is a record the mesh keeps and resumes (novox/hq ADR 0162): written whole, read back open,
// advanced, and gone from the open ones when done.
func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}, {"shop"}},
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}, "shop": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
open, err := inv.OpenPlans(ctx)
if err != nil || len(open) != 1 || open[0].ID != "plan-1" || len(open[0].Tiers) != 3 {
t.Fatalf("the plan was not kept whole: %v %+v", err, open)
}
// Another controller picks it up where it was left: a tier advanced and a module built.
now := time.Now().UTC()
resumed := open[0]
resumed.Tier = 1
resumed.Modules["mesh-tools"].State = "built"
resumed.Modules["mesh-tools"].BuiltAt = &now
resumed.State = PlanRolling
resumed.Note = "tier 0 built; waiting for builder on anchor to be applied"
if err := inv.SavePlan(ctx, &resumed); err != nil {
t.Fatal(err)
}
again, err := inv.PlanByID(ctx, "plan-1")
if err != nil || again.Tier != 1 || again.Modules["mesh-tools"].State != "built" || again.State != PlanRolling {
t.Fatalf("the advanced plan did not come back as left: %v %+v", err, again)
}
again.State = PlanDone
if err := inv.SavePlan(ctx, &again); err != nil {
t.Fatal(err)
}
if open, _ = inv.OpenPlans(ctx); len(open) != 0 {
t.Fatalf("a done plan is not open: %+v", open)
}
if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanDone {
t.Fatalf("a done plan is still among the recent ones: %+v", recent)
}
}
// novox/hq issue 254: a plan keeps the branch its merge went into, and a superseded plan is not open.
func TestASupersededPlanIsNotOpen(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc",
Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}},
Modules: map[string]*PlanModule{"gitea": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
kept, err := inv.PlanByID(ctx, "plan-1")
if err != nil || kept.Branch != "main" {
t.Fatalf("the branch was not kept: %v %+v", err, kept)
}
kept.State = PlanSuperseded
kept.Note = "superseded at tier 0 by plan-2"
if err := inv.SavePlan(ctx, &kept); err != nil {
t.Fatal(err)
}
if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 {
t.Fatalf("a superseded plan is still open: %v %+v", err, open)
}
if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanSuperseded {
t.Fatalf("a superseded plan is not among the recent ones as superseded: %+v", recent)
}
}
// novox/hq issue 349 (review of the follow-up): the newest merge of a branch is the one merged last, whatever
// order the plans were made in, in any state; a tie is broken by the plan made last; a release, another
// branch and another repository are never it; and its time is kept to the nanosecond.
func TestTheNewestMergeOfABranchIsTheOneMergedLast(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC)
save := func(id, repo, branch string, merged, created time.Time, state string, release bool) {
t.Helper()
p := Plan{ID: id, Repository: repo, Branch: branch, Commit: id + "-commit", Merged: merged, Created: created,
State: state, Tiers: [][]string{}, Modules: map[string]*PlanModule{}}
if release {
p.Release = &PlanRelease{}
}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
}
if _, found, err := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); err != nil || found {
t.Fatalf("a branch with no plan: %v %v", found, err)
}
// Made in the other order than merged: the later merge's plan made first, and done.
save("plan-later", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0, PlanDone, false)
save("plan-earlier", "novox/mesh-catalog", "main", t0.Add(time.Minute), t0.Add(5*time.Minute), PlanRolling, false)
save("plan-other-branch", "novox/mesh-catalog", "release", t0.Add(time.Hour), t0, PlanRolling, false)
save("plan-other-repo", "novox/mesh-controller", "main", t0.Add(time.Hour), t0, PlanRolling, false)
save("release-1", "novox/mesh-catalog", "main", t0.Add(time.Hour), t0, PlanRolling, true)
save("plan-unknown", "novox/mesh-catalog", "main", time.Time{}, t0.Add(time.Hour), PlanRolling, false)
p, found, err := inv.NewestMergeOf(ctx, "Novox/Mesh-Catalog", "main")
if err != nil || !found || p.ID != "plan-later" {
t.Fatalf("the newest merge: %s %v %v", p.ID, found, err)
}
if !p.Merged.Equal(t0.Add(2*time.Minute + 250*time.Millisecond)) {
t.Fatalf("its merge time was not kept to the nanosecond: %s", p.Merged)
}
// The same merge time: the plan made last.
save("plan-again", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0.Add(time.Minute), PlanBuilding, false)
if p, _, _ := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); p.ID != "plan-again" {
t.Fatalf("one merge time, two plans: %s", p.ID)
}
}
// novox/hq issue 352: what a machine was last sent is read back by name with its sequence, a report keeps
// the declaration it was about and that declaration's sequence, and a gate's sends are kept with the plan.
func TestASendAndAReportAreKnownByTheirDeclaration(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
record, err := inv.AddNode(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if _, found, err := inv.SentTo(ctx, "anchor"); err != nil || found {
t.Fatalf("a machine never sent anything: %v %v", found, err)
}
if _, _, err := inv.SentTo(ctx, "nobody"); err == nil {
t.Fatal("a machine that does not exist was answered")
}
seq, err := inv.NextSequence(ctx, record.ID)
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d-1", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
sent, found, err := inv.SentTo(ctx, "anchor")
if err != nil || !found || sent.Digest != "d-1" || sent.Sequence != seq {
t.Fatalf("sent %+v %v %v", sent, found, err)
}
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Node: "anchor", Outcome: OutcomeApplied, Declared: "d-1", Applied: 1},
ReportOrder{Sequence: seq, ReportSequence: 1}, func(ReportOrder) bool { return false }); err != nil {
t.Fatal(err)
}
reports, err := inv.LastReports(ctx)
if err != nil || len(reports) != 1 || reports[0].Declared != "d-1" || reports[0].ReportedSequence != seq || !reports[0].Current {
t.Fatalf("reports %+v %v", reports, err)
}
// Sent again, unreported: the report is no longer on the last send, and is still on the first.
if err := inv.RecordSent(ctx, record.ID, "d-2", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
reports, _ = inv.LastReports(ctx)
if reports[0].Current || !sent.ReportsOn(reports[0]) {
t.Fatalf("after a newer send: %+v", reports[0])
}
at := time.Now().UTC()
p := Plan{ID: "plan-352", Repository: "novox/x", Commit: "c", Created: at, State: PlanRolling, Tiers: [][]string{{"app"}},
Modules: map[string]*PlanModule{"app": {Gate: &PlanGate{Machines: []string{"anchor"}, Since: &at,
Sent: map[string]SentDeclaration{"anchor": sent}}}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
kept, err := inv.PlanByID(ctx, "plan-352")
if err != nil || !reflect.DeepEqual(kept.Modules["app"].Gate.Sent["anchor"], sent) {
t.Fatalf("the gate's send was not kept with the plan: %+v %v", kept.Modules["app"].Gate, err)
}
}
// A controller build's reach of the store's schema is kept by its version, and the store's own is read.
func TestASchemaReachIsKeptByBuild(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
applied, err := inv.SchemaApplied(ctx)
if err != nil || applied < 87 {
t.Fatalf("applied %d %v", applied, err)
}
if _, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || known {
t.Fatalf("an unrecorded build: %v %v", known, err)
}
if err := inv.RecordSchemaReach(ctx, "", 87); err == nil {
t.Fatal("a reach without a build was recorded")
}
if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 86); err != nil {
t.Fatal(err)
}
if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 87); err != nil {
t.Fatal(err)
}
if reach, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || !known || reach != 87 {
t.Fatalf("reach %d %v %v", reach, known, err)
}
}
// A plan keeps its backups through the store, timestamps to the nanosecond (novox/hq ADR 0404, issue 1065).
func TestAPlanKeepsItsBackupsInTheStore(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
at := time.Now().UTC()
p := Plan{ID: "plan-backups", Repository: "novox/mesh-catalog", Commit: "abc", Created: at, State: PlanBuilding,
Tiers: [][]string{{"mail"}}, Modules: map[string]*PlanModule{"mail": {}},
Backups: &PlanBackups{State: BackupPending, NoHolder: []string{"bare"},
Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupPending, AskedAt: &at}}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
got, err := inv.PlanByID(ctx, p.ID)
if err != nil || got.Backups == nil || !reflect.DeepEqual(got.Backups, p.Backups) {
t.Fatalf("kept %+v (%v), wrote %+v", got.Backups, err, p.Backups)
}
got.Backups.State = BackupDone
if err := inv.SavePlan(ctx, &got); err != nil {
t.Fatal(err)
}
if again, _ := inv.PlanByID(ctx, p.ID); again.Backups == nil || again.Backups.State != BackupDone {
t.Fatalf("the change was not kept: %+v", again.Backups)
}
}
internal/inventory/provisioned.go
-83
View File
@@ -1,8 +1,6 @@
package inventory
import (
"fmt"
"sort"
"time"
)
@@ -37,84 +35,3 @@ const (
ProvFailed = "failed"
ProvRemoved = "removed"
)
// ProvLookup answers a client's newest report of a module on a machine; false when it has said nothing of it.
type ProvLookup func(node, module string) (ProvReport, bool)
// OutcomeProvisioningFailed begins the outcome of a send whose module's files the provisioner failed to place.
const OutcomeProvisioningFailed = "provisioning failed"
// ProvisionedSend is a machine's report of a send, with its provisioner client's reports of every module the send
// handed to it: what a walk counts as that machine's report. engine and reported are the machine's own report and
// whether it said one. A failed client report is the send's outcome, whatever the machine said; none, or one not
// yet applied on the build sent, leaves the send unreported.
func ProvisionedSend(node string, send SentDeclaration, engine AppliedReport, reported bool, prov ProvLookup) (AppliedReport, bool) {
if len(send.Provisioned) == 0 {
return engine, reported
}
// The machine's own failure, or its wait for a busy container, is its report whatever the client says.
if reported && (engine.Outcome != OutcomeApplied || engine.Waiting) {
return engine, true
}
modules := make([]string, 0, len(send.Provisioned))
for m := range send.Provisioned {
modules = append(modules, m)
}
sort.Strings(modules)
at, placed := engine.At, true
for _, m := range modules {
r, said := ProvReport{}, false
if prov != nil {
r, said = prov(node, m)
}
switch {
case said && r.Status == ProvFailed && aboutTheSend(r, send, send.Provisioned[m]):
return AppliedReport{At: r.At, Outcome: fmt.Sprintf("%s (%s: %s)", OutcomeProvisioningFailed, m,
orNoReason(r.Reason))}, true
case said && r.Status == ProvApplied && r.Templates == send.Provisioned[m]:
if r.At.After(at) {
at = r.At
}
default:
placed = false
}
}
if !reported || !placed {
return AppliedReport{}, false
}
engine.At = at
return engine, true
}
// Only is the send narrowed to what it handed to the provisioner for the named modules: a delivery waits on the
// modules it carries, never on another module the same machine's declaration hands (ADR 0362 §1-2, issue 840).
// A send handing nothing is returned as it is.
func (s SentDeclaration) Only(modules ...string) SentDeclaration {
if len(s.Provisioned) == 0 {
return s
}
kept := map[string]string{}
for _, m := range modules {
if d, ok := s.Provisioned[m]; ok {
kept[m] = d
}
}
s.Provisioned = kept
return s
}
// aboutTheSend says a client's failure is about what the send handed: on the build's templates, or, said before it
// had any to name (its templates could not be fetched), no earlier than the send.
func aboutTheSend(r ProvReport, send SentDeclaration, digest string) bool {
if r.Templates != "" {
return r.Templates == digest
}
return send.At.IsZero() || !r.At.Before(send.At)
}
func orNoReason(s string) string {
if s == "" {
return "no reason given"
}
return s
}
internal/inventory/provisioned_test.go
-124
View File
@@ -1,124 +0,0 @@
package inventory
import (
"strings"
"testing"
"time"
)
// A walk counts a provisioned module applied on a machine only once its files are placed (novox/hq ADR 0362, design
// 57 P6, issue 672). The walk of issue 582 (one module, three machines of the rest), with g14's send having handed
// dunst to the provisioner: g14's own report comes 5 s after its send, shanks's, the last of the others, 15 s
// after its own.
const dunstTemplates = "sha256:" + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
func walkHandingDunst(t *testing.T) sendRest582 {
t.Helper()
c := walk582(t)
s := c.walk.Modules["build-agent"].Rest["g14"]
s.Provisioned = map[string]string{"build-agent": dunstTemplates}
c.walk.Modules["build-agent"].Rest["g14"] = s
return c
}
// provisioning is a walk's lookup with the machines' clients saying reports.
func (w sendRest582) provisioning(reports map[string]ProvReport) AppliedLookup {
return func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
r, ok := w.lookup(node, send, walkSent)
return ProvisionedSend(node, send, r, ok, func(node, module string) (ProvReport, bool) {
p, said := reports[node+"."+module]
return p, said
})
}
}
func restOf(w *WalkPhases, node string) RestReport {
for _, r := range w.Rest {
if r.Node == node {
return r
}
}
return RestReport{}
}
// The node-engine's report alone does not end the walk: it waits for prov.applied; with it, the walk ends, and
// its delivery time runs to the later of the two reports.
func TestAWalkWaitsForTheProvisionersReportAndEndsAtTheLater(t *testing.T) {
c := walkHandingDunst(t)
g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At
w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(nil))
if w.End != nil || restOf(w, "g14").State != RestWaiting ||
!strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") {
t.Fatalf("the walk did not wait for g14's prov.applied: end %v, g14 %+v, apply %+v", w.End, restOf(w, "g14"),
phase(w, PhaseApply, -1))
}
// A report on an older build of the templates is not this send's.
older := map[string]ProvReport{"g14.build-agent": {Status: ProvApplied, Templates: "sha256:" + strings.Repeat("0", 64),
At: g14Sent.Add(2 * time.Second)}}
if w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(older)); w.End != nil {
t.Fatalf("a report on an older build ended the walk at %v", w.End)
}
placed := g14Sent.Add(50 * time.Second) // after shanks's report, 35 s after the judgement
w = c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{
"g14.build-agent": {Status: ProvApplied, Templates: dunstTemplates, At: placed}}))
sumsUp(t, w)
if w.End == nil || !w.End.Equal(placed) {
t.Fatalf("the walk ends at %v, want the provisioner's report at %s: %+v", w.End, placed, w.Rest)
}
if g := restOf(w, "g14"); g.State != RestApplied || !g.Reported.Equal(placed) {
t.Fatalf("g14 is not applied at its later report: %+v", g)
}
}
// A failed prov.applied is the machine's failed apply, named in the walk's end with its reason.
func TestAFailedProvisioningIsAFailedApply(t *testing.T) {
c := walkHandingDunst(t)
g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At
w := c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{
"g14.build-agent": {Status: ProvFailed, Reason: "the templates cannot be fetched", At: g14Sent.Add(3 * time.Second)}}))
if w.End != nil {
t.Fatalf("a failed provisioning ended the walk: %v", w.End)
}
if g := restOf(w, "g14"); g.State != RestFailed || !strings.Contains(g.Outcome, "the templates cannot be fetched") {
t.Fatalf("g14 is not named failed with the reason: %+v", g)
}
if said := phase(w, PhaseApply, -1).Said; !strings.Contains(said, "not applied on g14 (provisioning failed (build-agent: "+
"the templates cannot be fetched))") {
t.Fatalf("the walk's end does not name g14's failure: %q", said)
}
// A failure said before the send, of no build, is not this send's.
stale := map[string]ProvReport{"g14.build-agent": {Status: ProvFailed, Reason: "old", At: g14Sent.Add(-time.Hour)}}
if g := restOf(c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(stale)), "g14"); g.State != RestWaiting {
t.Fatalf("a failure from before the send was read as this send's: %+v", g)
}
}
// None within ApplySilentAfter names the machine silent, never counted applied; the others end the walk.
func TestNoProvisioningReportWithinTheBoundIsSilence(t *testing.T) {
c := walkHandingDunst(t)
g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At
waiting := map[string]ProvReport{"g14.build-agent": {Status: ProvWaiting, WaitsFor: []string{"config:font-size"},
At: g14Sent.Add(3 * time.Second)}}
w := c.walk.WalkPhases(g14Sent.Add(ApplySilentAfter+time.Minute), c.provisioning(waiting))
sumsUp(t, w)
if g := restOf(w, "g14"); g.State != RestSilent {
t.Fatalf("g14 is not silent past ApplySilentAfter: %+v", g)
}
if len(w.Silent) != 1 || w.Silent[0] != "g14" || w.End == nil || !w.End.Equal(c.judged.Add(35*time.Second)) {
t.Fatalf("the walk does not end on the others, naming g14 silent: end %v, silent %v", w.End, w.Silent)
}
}
// A send that handed nothing ends on the node-engine's report alone, as before.
func TestASendThatHandedNothingEndsOnTheEnginesReport(t *testing.T) {
c := walk582(t)
before := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
now := c.walk.WalkPhases(c.judged.Add(time.Hour), c.provisioning(nil))
if before.End == nil || now.End == nil || !now.End.Equal(*before.End) {
t.Fatalf("a walk handing nothing ends differently: before %v, now %v", before.End, now.End)
}
}
internal/inventory/registered_held_integration_test.go
-59
View File
@@ -1,59 +0,0 @@
//go:build integration
package inventory
import (
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
manifestOf := func(image string) catalogue.Manifest {
return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
{"id": "app", "type": "container", "name": "one", "image": image}}}
}
record := func(id, image string, asked time.Time) Build {
t.Helper()
raw, _ := json.Marshal(manifestOf(image))
b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw,
Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
return b
}
x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111"
y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222"
before := record("x", x, time.Now().Add(-2*time.Hour))
failed := record("y", y, time.Now().Add(-time.Minute))
if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit,
Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil {
t.Fatal(err)
}
if err := inv.RestoreModule(ctx, before); err != nil {
t.Fatal(err)
}
held, err := inv.Held(ctx)
if err != nil {
t.Fatal(err)
}
running, err := inv.RegisteredHeld(ctx)
if err != nil {
t.Fatal(err)
}
if running["one/one"] == y {
t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"])
}
if got := running["one/one"]; got != "" && got != x {
t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x)
}
}
internal/inventory/sends.go
-364
View File
@@ -1,364 +0,0 @@
package inventory
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// The assignment generation and the record of every send (novox/hq issue 234, migration 0094).
//
// A declaration carries the generation of the assignments it was composed from, and a machine refuses one
// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later
// send carrying an older view. And every send is written down with who sent it, from which generation and
// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at.
// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes
// on a busy machine; the record is for finding who sent what, not an archive.
const sendsKept = 200
// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same
// transaction as every change to what is assigned where (a trigger on the assignment table).
func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) {
var g int64
if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil {
return 0, fmt.Errorf("reading the assignment generation: %w", err)
}
return g, nil
}
// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more
// than that, or what it already was when it is past it. It never lowers it.
//
// For one case only: a machine refused a send composed from the generation the mesh holds now, so the
// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after
// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed
// from the assignments the store holds now. A stale sender never meets this: its generation is below the
// counter, and the counter is not touched for it.
func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) {
var g int64
err := i.store.Pool().QueryRow(ctx,
`update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g)
if err != nil {
return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err)
}
return g, nil
}
// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the
// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed.
func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) {
var g *int64
if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil {
return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err)
}
if g == nil {
return 0, nil
}
return *g, nil
}
// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id.
func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) {
var reads bool
if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil {
return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err)
}
return reads, nil
}
// RecordReadsGeneration keeps what a machine's latest report said of reading a generation.
func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error {
_, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads)
return err
}
// ReadsModules says a machine's node-engine said it reads a module per resource and a build per module (issue 1145).
func (i *Inventory) ReadsModules(ctx context.Context, id string) (bool, error) {
var reads bool
if err := i.store.Pool().QueryRow(ctx, `select reads_modules from node where id = $1`, id).Scan(&reads); err != nil {
return false, fmt.Errorf("reading whether %s reads modules: %w", id, err)
}
return reads, nil
}
// RecordReadsModules keeps what a machine's latest report said of reading modules.
func (i *Inventory) RecordReadsModules(ctx context.Context, id string, reads bool) error {
_, err := i.store.Pool().Exec(ctx, `update node set reads_modules = $2 where id = $1`, id, reads)
return err
}
// Send is one declaration sent to a machine, as the mesh records it.
type Send struct {
// Node is the machine's id; NodeName its name, filled where it is read back.
Node string
NodeName string
Sequence int64
// Epoch is the lease epoch its sender acted under, zero for one that acted under none.
Epoch int64
// Sender is who sent it, in words: the caller, and the process and build that composed it.
Sender string
// Generation is the assignment generation it was composed from, zero when not known.
Generation int64
Digest string
// Modules are the modules it named.
Modules []string
// Provisioning is the provisioner's plan of each module the send handed to it, by module, as JSON (novox/hq
// ADR 0349 §4, design 57 P6): kept on the machine's record with the send, nil when it handed none. Never in
// the record of the send itself.
Provisioning []byte
SentAt time.Time
// RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the
// generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to
// because the refusal showed it behind the machine, zero when it was not.
RefusedAt *time.Time
RefusedApplied int64
CounterRaisedTo int64
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
Recorded bool
// Answer is what the machine last reported of this send; filled only where a last send is read back
// (LastSend, LastSends), the zero answer elsewhere.
Answer SendAnswer
}
// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
type SendAnswer struct {
// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
Outcome string
// Refused is the machine's own words when it refused the declaration whole.
Refused string
// Failed is how many of its resources failed, when some did.
Failed int
// At is when the machine reported it; nil when it has not.
At *time.Time
}
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " +
"hand, before sends were recorded, or by a controller whose record was not written)"
// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it
// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the
// caller says this loudly and does not take the send for failed.
type SendNotRecordedError struct{ Err error }
func (e *SendNotRecordedError) Error() string {
return "the record of the send was not written: " + e.Err.Error()
}
func (e *SendNotRecordedError) Unwrap() error { return e.Err }
// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and
// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue
// 234), in one transaction. The digest and the generation are written together, so the would-send is never
// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if
// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest
// sends beyond the newest 200 for the machine are let go.
func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64,
generation int64, s Send) error {
var sentEpoch *int64
if epoch > 0 {
e := int64(epoch)
sentEpoch = &e
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx,
`update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4,
sent_provisioning = $5::jsonb
where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil {
return err
}
recordErr := recordSend(ctx, tx, node, s)
if err := tx.Commit(ctx); err != nil {
return err
}
// And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record,
// for what a machine was sent is read from there.
if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil {
return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err)
}
if recordErr != nil {
return &SendNotRecordedError{Err: recordErr}
}
return nil
}
// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written.
func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error {
sp, err := tx.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = sp.Rollback(ctx) }()
modules := s.Modules
if modules == nil {
modules = []string{}
}
if _, err := sp.Exec(ctx,
`insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules)
values ($1, $2, $3, $4, $5, $6, $7)`,
node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest,
modules); err != nil {
return err
}
if err := pruneSends(ctx, sp, node); err != nil {
return err
}
return sp.Commit(ctx)
}
// pruneSends lets go of a machine's sends beyond the newest 200.
func pruneSends(ctx context.Context, q queries, node string) error {
_, err := q.Exec(ctx,
`delete from declaration_send where node = $1 and id not in
(select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept)
return err
}
// sendColumns are a send's columns as scanSends reads them.
const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0),
s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0),
s.recorded`
func scanSends(rows pgx.Rows) ([]Send, error) {
defer rows.Close()
var out []Send
for rows.Next() {
var s Send
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}
// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
sends, err := i.lastSends(ctx, `n.name = $1`, name)
if err != nil {
return Send{}, false, err
}
s, found := sends[name]
return s, found, nil
}
// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
return i.lastSends(ctx, `true`)
}
// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
// own. A periodic report about an identical earlier send names the same digest and may arrive after a later
// send the machine never received; only the sequence tells them apart. It also holds for a fast machine
// whose report lands before the send's row is stamped.
// - from an engine that reports no order, the report came after the send — both times the store's own — so
// a machine sent again a declaration it once applied does not read as having applied the new send.
func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
from declaration_send s join node n on n.id = s.node
left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
and case when r.reported_sequence is not null
then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
else r.at >= s.sent_at end
where s.recorded and `+where+`
order by n.name, s.id desc`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]Send{}
for rows.Next() {
var s Send
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
return nil, err
}
out[s.NodeName] = s
}
return out, rows.Err()
}
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal
// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may
// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none.
func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) {
rows, err := i.store.Pool().Query(ctx, `with refused as (
update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4
where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded
order by id desc limit 1)
returning *)
select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied,
nullIfZero(raisedTo))
if err != nil {
return Send{}, false, err
}
sends, err := scanSends(rows)
if err != nil || len(sends) == 0 {
return Send{}, false, err
}
return sends[0], true, nil
}
// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its
// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can
// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the
// refused declaration carried, and the refusal's applied generation and counter raise.
func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) {
rows, err := i.store.Pool().Query(ctx, `with refused as (
insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied,
counter_raised_to, recorded)
values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *)
select `+sendColumns+` from refused s join node n on n.id = s.node`,
s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest,
s.RefusedApplied, nullIfZero(s.CounterRaisedTo))
if err != nil {
return Send{}, err
}
sends, err := scanSends(rows)
if err != nil {
return Send{}, err
}
if len(sends) == 0 {
return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence)
}
if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil {
return Send{}, err
}
return sends[0], nil
}
// RefusedSendsSince is every send refused for its generation since a moment, newest first.
func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) {
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
from declaration_send s join node n on n.id = s.node
where s.refused_at >= $1 order by s.refused_at desc`, since)
if err != nil {
return nil, err
}
return scanSends(rows)
}
// nullIfEmpty is raw JSON, or null for none.
func nullIfEmpty(raw []byte) *string {
if len(raw) == 0 {
return nil
}
text := string(raw)
return &text
}
internal/inventory/sends_test.go
-342
View File
@@ -1,342 +0,0 @@
//go:build integration
package inventory
import (
"errors"
"slices"
"testing"
"time"
)
// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration
// newer in sequence than every other named four fewer modules than the assignments held, a machine applied
// it, and nothing the mesh kept could say who sent it or from what view.
func generationNow(t *testing.T, inv *Inventory) int64 {
t.Helper()
g, err := inv.AssignmentGeneration(t.Context())
if err != nil {
t.Fatal(err)
}
return g
}
// **Every change to what is assigned where raises the generation** — an assignment and an unassignment — and a repeated
// assignment, which changes nothing, does not.
func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) {
inv, node := aNodeWithModules(t, "postgres", "web")
ctx := t.Context()
start := generationNow(t, inv)
if start < 1 {
t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start)
}
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
assigned := generationNow(t, inv)
if assigned <= start {
t.Fatalf("an assignment left the generation at %d", assigned)
}
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
if again := generationNow(t, inv); again != assigned {
t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again)
}
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
unassigned := generationNow(t, inv)
if unassigned <= assigned {
t.Fatalf("an unassignment left the generation at %d", unassigned)
}
// A machine's removal, which no verb makes, is read from the machines the store knows: an assignment on a machine it
// does not know is none (assignments in the bucket have no key of the store to cascade through).
}
// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter
// is behind it** — a store put back from a backup — so the next send is not refused for ever.
func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) {
inv, _ := aNodeWithModules(t)
now := generationNow(t, inv)
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 {
t.Fatalf("raised past %d to %d (%v)", now+40, raised, err)
}
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 {
t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err)
}
}
// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it
// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation
// written with the digest, and a refusal is kept on the send it refused, naming its sender.
func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)",
Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}}
if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil {
t.Fatal(err)
}
stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor",
Generation: 38, Digest: "d12", Modules: []string{"docker"}}
if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil {
t.Fatal(err)
}
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not kept: %v", err)
}
if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded ||
!slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() {
t.Fatalf("the last send reads %+v", last)
}
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 {
t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err)
}
refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0)
if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 ||
refused.CounterRaisedTo != 0 {
t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err)
}
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 {
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
}
if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found {
t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err)
}
}
// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it;
// it is not the machine's last send, and it says what the counter was raised to.
func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41,
Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45})
if err != nil {
t.Fatal(err)
}
if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 ||
kept.NodeName != node {
t.Fatalf("the refusal reads %+v", kept)
}
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(since) != 1 || since[0].Sequence != 99 {
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
}
if _, found, err := inv.LastSend(ctx, node); err != nil || found {
t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err)
}
}
// **The machine's record and the send's are written together, and a send record that cannot be written
// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a
// send that is away must not read as failed, nor the machine as behind.
func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
// A NUL byte is text PostgreSQL refuses: the send's own row cannot be written.
broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"}
err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken)
var unrecorded *SendNotRecordedError
if !errors.As(err, &unrecorded) {
t.Fatalf("a send record that could not be written read as %v", err)
}
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 {
t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err)
}
if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" {
t.Fatalf("the machine's digest was not written: %q (%v)", sent, err)
}
}
// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer.
func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
inv, node := aNodeWithModules(t)
record, err := inv.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
for _, reads := range []bool{false, true, false} {
if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil {
t.Fatal(err)
}
if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads {
t.Fatalf("recorded %v, read %v (%v)", reads, got, err)
}
}
}
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
// words, or not answered yet — read from the machine's report only when that report is about the send's
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
}
report := func(d Doing) {
t.Helper()
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
t.Fatal(err)
}
}
answer := func() SendAnswer {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer
}
send(11, "d11")
if a := answer(); a.Outcome != "" || a.At != nil {
t.Fatalf("a send not reported on reads as answered: %+v", a)
}
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
t.Fatalf("a send the machine reported applying reads %+v", a)
}
send(12, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
}
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
t.Fatalf("a send the machine refused reads %+v", a)
}
// Sent again a declaration it reported applying before the send: not answered until it reports again.
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
send(13, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
}
all, err := inv.LastSends(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
t.Fatalf("every machine's last send reads %+v", all)
}
if _, found := all["laptop"]; found {
t.Error("a machine never sent anything has a last send")
}
}
// **A report about another declaration, or another send of the same one, is not the last send's answer**
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
// the report's epoch and sequence say which send it is about.
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
t.Fatal(err)
}
}
ordered := func(epoch, sequence int64, digest string) {
t.Helper()
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
t.Fatal(err)
}
}
answer := func() string {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer.Outcome
}
// An engine that reports no order: the digest decides.
send(11, "d11")
send(12, "d12")
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
t.Fatal(err)
}
if a := answer(); a != "" {
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
}
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
send(20, "d20")
ordered(57, 20, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
}
send(21, "d20")
ordered(57, 20, "d20")
if a := answer(); a != "" {
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
}
ordered(56, 21, "d20")
if a := answer(); a != "" {
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
}
ordered(57, 21, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
}
}
// 1145/AC3: whether a machine reads modules is its latest word, as whether it reads a generation is.
func TestWhetherAMachineReadsModulesIsItsLatestWord(t *testing.T) {
inv, node := aNodeWithModules(t)
record, err := inv.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
for _, reads := range []bool{false, true, false} {
if err := inv.RecordReadsModules(t.Context(), record.ID, reads); err != nil {
t.Fatal(err)
}
if got, err := inv.ReadsModules(t.Context(), record.ID); err != nil || got != reads {
t.Fatalf("1145/AC3: recorded %v, read %v (%v)", reads, got, err)
}
}
}
internal/inventory/sent_builds_test.go
-86
View File
@@ -1,86 +0,0 @@
//go:build integration
package inventory
import (
"reflect"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent
// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which
// is not the same as having been sent nothing.
func TestASendKeepsTheBuildsItCarried(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
node, err := inv.AddNode(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err)
}
carried := map[string]string{"resolver": "abc123", "network": ""}
if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil {
t.Fatal(err)
}
// What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried
// with no commit known names none.
builds, known, err := inv.SentBuilds(ctx, "anchor")
if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) {
t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err)
}
// Sent with nothing carried: known, and empty.
if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 {
t.Fatalf("an empty send: %v %v %v", builds, known, err)
}
// Sent by hand: not known, and the digest still recorded.
if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err)
}
if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" {
t.Fatalf("the digest was not recorded with it: %q %v", sent, err)
}
if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known {
t.Fatalf("a machine the mesh does not know: %v %v", known, err)
}
}
// A module's current build is the commit its manifest was read at, with its upgrade policy.
func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"},
Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil {
t.Fatal(err)
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
t.Fatal(err)
}
// Rolled out by default, one machine first and gated (novox/hq ADR 0236).
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true, From: catalogue.FromDefault}) {
t.Errorf("resolver is at %+v", got)
}
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true, From: catalogue.FromPerson}) {
t.Errorf("a module with no source is at %+v", got)
}
}
internal/inventory/sent_bus_users_test.go
-27
View File
@@ -1,27 +0,0 @@
//go:build integration
package inventory
import "testing"
// novox/hq issue 249: the digest of the user list a machine was last sent is kept, by its name, and
// is empty for a machine never sent one.
func TestTheUserListAMachineWasSentIsKept(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
t.Fatal(err)
}
if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "" {
t.Fatalf("a machine never sent a list has %q: %v", sent, err)
}
if err := inv.RecordSentBusUsers(ctx, "anchor", "abc"); err != nil {
t.Fatal(err)
}
if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "abc" {
t.Fatalf("the list sent was not kept: %q %v", sent, err)
}
if sent, err := inv.SentBusUsers(ctx, "nobody"); err != nil || sent != "" {
t.Fatalf("a machine the mesh does not know: %q %v", sent, err)
}
}
internal/inventory/sent_plans_test.go
-95
View File
@@ -1,95 +0,0 @@
//go:build integration
package inventory
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
// A send keeps the provisioner's plans it handed (design 57 P6, issue 672, migration 0100): the assignments mirror
// carries the plan of the build that machine was sent, in the module's own assignment and no other, and a walk
// reads which templates the send handed.
func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) {
inv, node := aNodeWithModules(t, "dunst", "web")
ctx := t.Context()
for _, m := range []string{"dunst", "web"} {
if _, err := inv.Assign(ctx, node, m); err != nil {
t.Fatal(err)
}
}
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
digest := "sha256:" + strings.Repeat("d", 64)
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
// A plan for a module the machine no longer has assigned is kept with the send and mirrored nowhere; one for a
// module only pulled in by another is mirrored under its own key, saying what requires it (issue 765).
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"gone":` + mustJSON(t, plan) + `,"notify":` +
strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
t.Fatal(err)
}
sent, _, err := inv.SentTo(ctx, node)
if err != nil || !reflect.DeepEqual(sent.Provisioned, map[string]string{"dunst": digest, "gone": digest,
"notify": digest}) {
t.Fatalf("the send says it handed %v (%v)", sent.Provisioned, err)
}
plans, err := inv.SentProvisioningPlans(ctx)
required := plan
required.RequiredBy = "desk"
if err != nil || !reflect.DeepEqual(plans, map[string]Provisioning{node + ".dunst": plan, node + ".notify": required}) {
t.Fatalf("the plans mirrored are %+v (%v)", plans, err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
mirror := inv.PlanValuesForTest(t)
var dunst, web AssignmentMirrored
if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(mirror[node+".web"], &web); err != nil {
t.Fatal(err)
}
if dunst.Provisioning == nil || !reflect.DeepEqual(*dunst.Provisioning, plan) || web.Provisioning != nil {
t.Fatalf("the mirror carries %+v for dunst and %+v for web", dunst.Provisioning, web.Provisioning)
}
var notify AssignmentMirrored
if err := json.Unmarshal(mirror[node+".notify"], &notify); err != nil {
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror[node+".notify"], err)
}
if notify.RequiredBy != "desk" || !notify.Assigned.IsZero() || notify.Provisioning == nil ||
!reflect.DeepEqual(*notify.Provisioning, plan) {
t.Fatalf("the mirror carries %s for the pulled-in module", mirror[node+".notify"])
}
if _, has := mirror[node+".gone"]; has {
t.Fatalf("a module no longer assigned is mirrored: %s", mirror[node+".gone"])
}
// The next send hands nothing: nothing is kept, and the assignment's value is the bytes it was.
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second"}); err != nil {
t.Fatal(err)
}
if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil {
t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err)
}
if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") {
t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"])
} else if _, has := mirror[node+".notify"]; has {
t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"])
}
}
func mustJSON(t *testing.T, v any) string {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
internal/inventory/source.go
+1 -5
View File
@@ -30,7 +30,6 @@ type sourceRow struct {
made []byte
manifest []byte
manifestDigest string
failedGate bool
}
// sourceRows is every successful build of each module (of one module, when named), newest request
@@ -53,13 +52,10 @@ func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][
func standing(builds []sourceRow, from int) int {
at := from
fp := builds[from].fingerprint
if fp == "" || builds[from].failedGate {
if fp == "" {
return at
}
for j := from + 1; j < len(builds); j++ {
if builds[j].failedGate {
break
}
if builds[j].fingerprint != fp {
if !newForm(fp) || newForm(builds[j].fingerprint) || !sameDigests(builds[at].made, builds[j].made) {
break
internal/inventory/stored_silent_file_test.go
+5 -1
View File
@@ -39,10 +39,14 @@ func TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed(t *testing.T
t.Fatal(err)
}
m, found, err := inv.ManifestAt(ctx, "mail-client", b.Commit)
kept, found, err := inv.BuildByID(ctx, b.ID)
if err != nil || !found {
t.Fatalf("the stored build is not read again: found %v, %v", found, err)
}
m, err := catalogue.ParseManifest(kept.Manifest)
if err != nil {
t.Fatalf("the stored manifest is not read again: %v", err)
}
if written := catalogue.WrittenSettingsProblems(m); len(written) != 1 {
t.Fatalf("the rule this build predates should name its accounts file where it is written: %v", written)
}
internal/inventory/tier_entered_resume_1886_test.go
-32
View File
@@ -1,32 +0,0 @@
package inventory
import (
"testing"
"time"
)
// Issue 1886: R4e entered tier 1 at 02:01, was stopped, and was resumed at 07:43 at the same tier. Its save kept the
// tier's entry of 02:01, so at 07:50 the plan read "waiting for mesh-build-image on shanks to be applied for 5h48m53s"
// and its progress watchdog measured the tier from before the stop. A walk set open again at the tier it ended at
// enters that tier anew: its time in the tier runs from the resume.
func TestAWalkResumedAtItsTierEntersItAnew(t *testing.T) {
entered := time.Date(2026, 10, 23, 0, 1, 7, 0, time.UTC)
resumed := time.Date(2026, 10, 23, 5, 43, 30, 0, time.UTC)
old := &Plan{ID: "plan-r4e", State: PlanFailed, Tier: 1, Tiers: [][]string{{"a"}, {"b"}, {"c"}},
Created: entered.Add(-2 * time.Hour), TierEntered: entered}
p := *old
p.State = PlanBuilding
got, d := tierLeftOf(old, p, resumed)
if !got.Equal(resumed) || d != nil {
t.Fatalf("1886: a walk resumed at tier 1 entered it at %s (duration %v); want %s, no tier left", got, d, resumed)
}
// A walk still in its tier keeps its entry; one that ended keeps it too.
open := *old
open.State = PlanRolling
if got, _ := tierLeftOf(&open, open, resumed); !got.Equal(entered) {
t.Fatalf("an open walk still in its tier was stamped anew: %s", got)
}
if got, _ := tierLeftOf(old, *old, resumed); !got.Equal(entered) {
t.Fatalf("a failed walk saved again was stamped anew: %s", got)
}
}
internal/inventory/waiting_merges_test.go
-33
View File
@@ -1,33 +0,0 @@
package inventory
import (
"context"
"testing"
"time"
)
// A merge no plan answers waits, and is listed as waiting until it is answered where it is kept (novox/hq issue 1146):
// what a first merge into a new repository does while it waits for mesh-delivery's word.
func TestAMergeNoPlanAnswersIsListedWaitingUntilItIsAnswered(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := context.Background()
now := time.Now().UTC()
if added, err := inv.chainAddMerge(ctx, c, BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "c1",
Merged: now, Heard: now}); err != nil || !added {
t.Fatalf("added %v, %v", added, err)
}
waiting, err := inv.chainWaitingMerges(ctx, c)
if err != nil || len(waiting) != 1 || waiting[0].Commit != "c1" {
t.Fatalf("waiting %+v, %v", waiting, err)
}
move := []MergeMove{{Repository: "novox/mesh-delivery", Commit: "c1", Plan: WaitingRecord}}
if err := inv.chainSavePlans(ctx, c, nil, move); err != nil {
t.Fatal(err)
}
if waiting, err = inv.chainWaitingMerges(ctx, c); err != nil || len(waiting) != 0 {
t.Fatalf("answered, it still waits: %+v, %v", waiting, err)
}
if _, known, err := inv.chainMergeOf(ctx, c, "novox/mesh-delivery", "c1"); err != nil || !known {
t.Fatalf("an answered merge is no longer kept: %v %v", known, err)
}
}
internal/inventory/walkphases.go
-676
View File
@@ -1,676 +0,0 @@
package inventory
import (
"context"
"fmt"
"sort"
"strings"
"time"
)
// Where a walk's time went (novox/hq ADR 0282 decision 6, issue 382): from its merge to every machine of its
// rest running its builds, phase by phase, worked out from the walk's own moments. Measurement only: nothing
// here decides anything about the walk.
//
// **A phase that cannot be measured is said unknown, never zero.** Where a moment is missing (a walk kept
// before it was recorded, a machine not yet reported), the phases around it are unknown, and the span between
// the moments on either side is counted as unknown time: the measured phases and the unknown time always add
// up to the total. A phase that did not happen (no window for a merge walked alone, no first machine for a
// module nobody runs) is said none, with no time.
// The phases, in the order a walk passes them (ADR 0282's table).
const (
PhaseWindow = "window" // the merge to its batch's window closing
PhaseQueued = "queued" // the window closed to the walk being cut: waiting behind another walk
PhaseWord = "word" // the cut to the delivery's word, for a walk that waits for one
PhaseBetween = "between-tiers" // one tier's end to the next tier's ask
PhaseBuild = "build" // a tier asked to its last module built
PhaseSend = "send-first" // built to sent to the first machines
PhaseBusy = "busy-wait" // sent first to a busy container there replaced, when one waited (ADR 0335)
PhaseJudge = "judgement" // sent first to the first-node gate's last verdict: its readings
PhaseRest = "send-rest" // judged to sent to the rest
PhaseApply = "apply" // the last send to every machine of the rest reporting the build applied
PhaseOpen = "open" // a walk still running: since its last moment
PhaseMeasured = "measured"
PhaseUnknown = "unknown"
PhaseNone = "none"
)
// The classes of a walk (ADR 0282 decision 1) and their delivery budgets.
const (
ClassCore = "core"
ClassLeaf = "leaf"
)
// ApplySilentAfter is how long after a send to the rest a machine that has said nothing is left out of the
// walk's end, as a machine not heard from (ADR 0282 decision 1: a sleeping laptop is listed, not waited for).
const ApplySilentAfter = 15 * time.Minute
// WalkPhases is a walk's time, phase by phase.
type WalkPhases struct {
Class string `json:"class,omitempty"`
// From is the walk's earliest merge; End when its last phase ended: every machine of its rest reported the
// build applied. Nil End while that is not known.
From *time.Time `json:"from,omitempty"`
End *time.Time `json:"end,omitempty"`
// TotalMS is End − From; zero while either is unknown, Total its words.
TotalMS int64 `json:"total_ms,omitempty"`
Total string `json:"total,omitempty"`
// UnknownMS is the time within the walk no phase could be measured over.
UnknownMS int64 `json:"unknown_ms,omitempty"`
Phases []WalkPhase `json:"phases"`
Silent []string `json:"silent,omitempty"`
// Rest is each machine of the rest's own report of the send that carried the walk's build there (novox/hq ADR
// 0282 decision 6, issue 582): when it was sent, when it said, and what.
Rest []RestReport `json:"rest,omitempty"`
Said string `json:"said"`
Merges []MergeStart `json:"merges,omitempty"`
}
// MergeStart is one merge the walk answers and when it was made: where that merge's delivery time starts.
type MergeStart struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Merged time.Time `json:"merged"`
}
// WalkPhase is one phase of a walk.
type WalkPhase struct {
Name string `json:"name"`
// Tier is the tier a tier's phase belongs to; -1 for a phase of the walk.
Tier int `json:"tier"`
State string `json:"state"`
Start *time.Time `json:"start,omitempty"`
End *time.Time `json:"end,omitempty"`
TookMS int64 `json:"took_ms,omitempty"`
Took string `json:"took,omitempty"`
Said string `json:"said,omitempty"`
}
// AppliedReport is a machine's first report, after a send, that it applied what it was sent.
type AppliedReport struct {
At time.Time
Outcome string
// Answered is, for a machine that waited for a busy container (novox/hq ADR 0335), its first report of the
// send; Busy how long it then waited; Waiting that it still waits, At being its first report then.
Answered time.Time
Busy time.Duration
Waiting bool
}
// AppliedLookup answers a machine's report of the send the walk made to it: send is what the walk kept of it, and
// walkSent the walk's own record of its send to the rest; false when it has not reported.
type AppliedLookup func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool)
// RestReport is one machine of the rest and its own report of a send of the walk (novox/hq issue 582).
type RestReport struct {
Node string `json:"node"`
Modules []string `json:"modules"`
// Sent is when that machine was sent, by its own send time where the walk kept it; else the walk's record of
// its send to the rest, and SentSaid says so.
Sent time.Time `json:"sent"`
SentSaid string `json:"sent_said,omitempty"`
Sequence int64 `json:"sequence,omitempty"`
Reported *time.Time `json:"reported,omitempty"`
Took string `json:"took,omitempty"`
Outcome string `json:"outcome,omitempty"`
// Busy is how long it waited for a busy container before it ran the build (novox/hq ADR 0335): inside Took,
// never read as the machine being silent.
Busy string `json:"busy,omitempty"`
// State is applied, failed, waiting (not yet heard, within ApplySilentAfter), busy (it answered and waits for
// a busy container: never silent) or silent (not heard within it: listed, never moving the walk's end).
State string `json:"state"`
busy time.Duration
}
// The states of a machine of the rest.
const (
RestApplied = "applied"
RestFailed = "failed"
RestWaiting = "waiting"
RestBusy = "busy"
RestSilent = "silent"
)
// point is one moment of the walk, ending the phase named.
type point struct {
phase string
tier int
at *time.Time
none bool // the phase did not happen
said string // why it is none, or unknown
}
// Phases is the walk's time phase by phase, at now; applied answers the rest's reports (nil: none read).
func (p Plan) WalkPhases(now time.Time, applied AppliedLookup) *WalkPhases {
if p.Release != nil || p.Batch() {
return nil
}
w := &WalkPhases{}
if p.Times != nil {
w.Class = p.Times.Class
}
from := p.firstMerge(w)
if from != nil {
f := from.Truncate(time.Millisecond)
from = &f
}
if from == nil {
w.Said = "when its merge was made is not kept: its delivery time is unknown"
} else {
w.From = from
}
points := p.points(now, applied, w)
// Walk the moments: each known moment after a known one is a measured phase; a missing moment makes the
// phases up to the next known one unknown, and their span unknown time.
last := from
var pending []int
for _, pt := range points {
if pt.none {
w.Phases = append(w.Phases, WalkPhase{Name: pt.phase, Tier: pt.tier, State: PhaseNone, Said: pt.said})
continue
}
ph := WalkPhase{Name: pt.phase, Tier: pt.tier, Said: pt.said}
if pt.at == nil {
ph.State = PhaseUnknown
w.Phases = append(w.Phases, ph)
pending = append(pending, len(w.Phases)-1)
continue
}
at := pt.at.Truncate(time.Millisecond)
ph.End = &at
if last != nil && at.Before(*last) {
// **Out of order** (issue 382): this moment was recorded before the one before it — a module's first
// send kept before its build was, two clocks, a late record. Neither phase can be measured honestly:
// the one before it is said unknown and its time joins the unknown time, and this one is said unknown
// with no time. The walk goes on from the later moment, so nothing is negative and the sum holds.
for i := len(w.Phases) - 1; i >= 0; i-- {
prev := &w.Phases[i]
if prev.End == nil {
continue // a phase that did not happen, or is unknown with no moment: the one before carries last
}
if prev.State == PhaseMeasured {
w.UnknownMS += prev.TookMS
}
prev.State, prev.Start, prev.TookMS, prev.Took = PhaseUnknown, nil, 0, ""
prev.Said = "out of order: the phase after it ended first"
break
}
ph.State, ph.Said = PhaseUnknown, "out of order: it ended before the phase before it"
w.Phases = append(w.Phases, ph)
pending = nil
continue
}
if last != nil && len(pending) == 0 {
start := *last
ph.State, ph.Start = PhaseMeasured, &start
ph.TookMS = at.Sub(start).Milliseconds()
ph.Took = words(at.Sub(start))
} else {
ph.State = PhaseUnknown
if last != nil {
w.UnknownMS += at.Sub(*last).Milliseconds()
}
}
w.Phases = append(w.Phases, ph)
pending = nil
last = &at
}
if len(pending) > 0 {
// The walk's last moments are unknown: it has no end.
if w.Said == "" {
w.Said = "its end is unknown: " + w.Phases[pending[0]].Name + " " + orNot(w.Phases[pending[0]].Said)
}
return w
}
if last == nil || from == nil {
return w
}
if p.Open() {
since := *last
w.Phases = append(w.Phases, WalkPhase{Name: PhaseOpen, Tier: -1, State: PhaseOpen, Start: &since,
TookMS: now.Sub(since).Milliseconds(), Took: words(now.Sub(since)), Said: "the walk is " + p.State})
w.Said = "open: " + p.State + ", " + words(now.Sub(*from)) + " since its merge"
return w
}
if p.State != PlanDone {
w.Said = "ended " + p.State + ": no delivery time"
return w
}
end := *last
w.End = &end
w.TotalMS = end.Sub(*from).Milliseconds()
w.Total = words(end.Sub(*from))
if w.Said == "" {
if w.UnknownMS > 0 {
w.Said = fmt.Sprintf("%s from its merge to running everywhere, %s of it unknown", w.Total,
words(time.Duration(w.UnknownMS)*time.Millisecond))
} else {
w.Said = w.Total + " from its merge to running everywhere"
}
}
return w
}
// firstMerge is when the walk's earliest merge was made, and every merge's start kept on w.
func (p Plan) firstMerge(w *WalkPhases) *time.Time {
var first *time.Time
if p.Delivery != nil {
for _, m := range p.Delivery.Merges {
if m.Merged.IsZero() {
continue
}
w.Merges = append(w.Merges, MergeStart{Repository: m.Repository, Commit: m.Commit, Merged: m.Merged})
if first == nil || m.Merged.Before(*first) {
at := m.Merged
first = &at
}
}
}
if first == nil {
for _, c := range p.Carried() {
if c.Merged.IsZero() {
continue
}
w.Merges = append(w.Merges, MergeStart{Repository: c.Repository, Commit: c.Commit, Merged: c.Merged})
if first == nil || c.Merged.Before(*first) {
at := c.Merged
first = &at
}
}
}
return first
}
// points are the walk's moments in order.
func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []point {
var out []point
// The window and the wait behind another walk.
cut := p.Created
if p.Times != nil && p.Times.Cut != nil {
cut = *p.Times.Cut
}
switch {
case p.Times == nil:
out = append(out, point{phase: PhaseWindow, tier: -1, said: "not kept for a walk made before ADR 0282"},
point{phase: PhaseQueued, tier: -1, at: &cut, said: "the window and the wait behind another walk together"})
case p.Times.WindowClosed == nil:
out = append(out, point{phase: PhaseWindow, tier: -1, none: true, said: "no window: walked on its own"},
point{phase: PhaseQueued, tier: -1, at: &cut})
default:
closed := *p.Times.WindowClosed
out = append(out, point{phase: PhaseWindow, tier: -1, at: &closed}, point{phase: PhaseQueued, tier: -1, at: &cut})
}
if p.Delivery != nil && p.Delivery.Awaits != "" {
out = append(out, point{phase: PhaseWord, tier: -1, at: p.Delivery.Go, said: "waiting for " + p.Delivery.Awaits + "'s word"})
} else {
out = append(out, point{phase: PhaseWord, tier: -1, none: true, said: "waits for no word"})
}
var lastSends []restSend
for t, tier := range p.Tiers {
if t > p.Tier || (t == p.Tier && p.Tier < len(p.Tiers) && !askedAnyOf(p, tier)) {
break
}
var asked, built, first, judged, rest, busyEnded *time.Time
allBuilt, anyFirst, allJudged, allRest := true, false, true, true
// A busy container on a first machine holds the judging until it is replaced (novox/hq ADR 0335 §4): a
// phase of its own, only in a tier where one waited, counted in the delivery time and never exempted.
anyBusy, allBusyEnded, busyOn := false, true, ""
for _, m := range tier {
s := p.Modules[m]
if s == nil {
allBuilt, allRest = false, false
continue
}
asked = earliest(asked, s.AskedAt)
if s.BuiltAt == nil {
if s.State != "deleted" {
allBuilt = false
}
} else {
built = latest(built, s.BuiltAt)
}
if s.FirstAt != nil {
anyFirst = true
first = earliest(first, s.FirstAt)
if s.Gate != nil && s.Gate.Busy != nil {
anyBusy = true
if s.Gate.Busy.Ended == nil {
allBusyEnded, busyOn = false, s.Gate.Busy.Machine
} else {
busyEnded = latest(busyEnded, s.Gate.Busy.Ended)
}
}
if s.Gate != nil {
if s.Gate.JudgedAt == nil {
allJudged = false
} else {
judged = latest(judged, s.Gate.JudgedAt)
}
}
}
if s.SentAt == nil {
if s.State != "deleted" {
allRest = false
}
} else {
rest = latest(rest, s.SentAt)
for node, what := range s.Rest {
lastSends = append(lastSends, restSend{module: m, node: node, walkSent: *s.SentAt, what: what})
}
}
}
if t > 0 {
out = append(out, point{phase: PhaseBetween, tier: t, at: asked})
}
if !allBuilt {
built = nil
}
out = append(out, point{phase: PhaseBuild, tier: t, at: built})
if anyFirst {
if !allJudged {
judged = nil
}
out = append(out, point{phase: PhaseSend, tier: t, at: first})
if anyBusy {
said := ""
if !allBusyEnded {
busyEnded, said = nil, "waits for a busy container on "+busyOn
}
out = append(out, point{phase: PhaseBusy, tier: t, at: busyEnded, said: said})
}
out = append(out, point{phase: PhaseJudge, tier: t, at: judged})
} else {
out = append(out, point{phase: PhaseSend, tier: t, none: true, said: "no first machine: nothing to judge"},
point{phase: PhaseJudge, tier: t, none: true, said: "no first machine: nothing to judge"})
}
if !allRest {
rest = nil
}
out = append(out, point{phase: PhaseRest, tier: t, at: rest})
}
if p.State != PlanDone {
return out
}
// Every machine of the rest running the build: its first report after the send, applied.
if p.Times == nil {
return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's sends are not kept for a walk made before ADR 0282"})
}
if len(lastSends) == 0 {
// novox/hq issue 720: since ADR 0360 no gate reads a walk; its one machine's report is its run.
return append(out, point{phase: PhaseApply, tier: -1, none: true,
said: "no machine beyond the first: its one machine's report was its run"})
}
if applied == nil {
return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's reports were not read"})
}
// Each machine of the rest, by its own report of the send that carried the build there (novox/hq issue 582):
// never a report of a send made after it, which another walk's send may be.
reports := map[string]*RestReport{}
var order []string
for _, s := range lastSends {
at, said := s.walkSent, ""
if !s.what.At.IsZero() {
at = s.what.At
} else {
said = "the walk's record of its send to the rest: this machine's own send time was not kept"
}
key := s.node + "\x00" + s.what.Digest + "\x00" + at.Format(time.RFC3339Nano)
if r := reports[key]; r != nil {
r.Modules = appendOnce(r.Modules, s.module)
continue
}
rr := &RestReport{Node: s.node, Modules: []string{s.module}, Sent: at, SentSaid: said, Sequence: s.what.Sequence}
// Judged on the modules this walk sent there, not on every module the machine's declaration hands (ADR 0362).
var sentThere []string
for _, o := range lastSends {
if o.node == s.node && o.what.Digest == s.what.Digest && o.what.At.Equal(s.what.At) {
sentThere = append(sentThere, o.module)
}
}
if r, ok := applied(s.node, s.what.Only(sentThere...), s.walkSent); ok {
got := r.At
rr.Reported, rr.Outcome, rr.Took = &got, r.Outcome, words(r.At.Sub(at))
if r.Busy > 0 {
rr.Busy = words(r.Busy)
rr.busy = r.Busy
}
if r.Waiting {
rr.Reported, rr.Took, rr.State = nil, "", RestBusy
}
}
reports[key] = rr
order = append(order, key)
}
// A machine sent twice by the walk (one tier's rest, then the next's) that said nothing of the first send has
// said it of the second, which carries the first's builds too: both are this walk's.
for _, k := range order {
rr := reports[k]
if rr.Reported != nil {
continue
}
for _, l := range order {
later := reports[l]
if later.Node == rr.Node && later.Sent.After(rr.Sent) && later.Reported != nil &&
(rr.Reported == nil || later.Reported.Before(*rr.Reported)) {
got := *later.Reported
rr.Reported, rr.Outcome, rr.Took, rr.State = &got, later.Outcome, words(got.Sub(rr.Sent)), ""
}
}
}
var end *time.Time
var waiting, failed, busy []string
for _, k := range order {
rr := reports[k]
switch {
case rr.State == RestBusy:
// It answered and waits for a busy container: never silent, and the walk's end waits for it.
busy = appendOnce(busy, rr.Node)
case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter,
rr.Reported != nil && rr.Reported.Sub(rr.Sent)-rr.busy > ApplySilentAfter:
rr.State = RestSilent
w.Silent = appendOnce(w.Silent, rr.Node)
case rr.Reported == nil:
rr.State = RestWaiting
waiting = appendOnce(waiting, rr.Node)
case rr.Outcome != OutcomeApplied:
rr.State = RestFailed
failed = appendOnce(failed, rr.Node+" ("+rr.Outcome+")")
default:
rr.State = RestApplied
end = latest(end, rr.Reported)
}
sort.Strings(rr.Modules)
w.Rest = append(w.Rest, *rr)
}
sort.SliceStable(w.Rest, func(a, b int) bool {
if w.Rest[a].Node != w.Rest[b].Node {
return w.Rest[a].Node < w.Rest[b].Node
}
return w.Rest[a].Sent.Before(w.Rest[b].Sent)
})
switch {
case len(failed) > 0:
return append(out, point{phase: PhaseApply, tier: -1, said: "not applied on " + strings.Join(failed, ", ")})
case len(waiting) > 0 || len(busy) > 0:
var said []string
if len(waiting) > 0 {
said = append(said, "waiting for "+strings.Join(waiting, ", ")+" to report it applied")
}
if len(busy) > 0 {
said = append(said, "waits for a busy container on "+strings.Join(busy, ", "))
}
return append(out, point{phase: PhaseApply, tier: -1, said: strings.Join(said, "; ")})
case end == nil:
return append(out, point{phase: PhaseApply, tier: -1, said: "no machine of the rest heard from: " +
strings.Join(w.Silent, ", ")})
}
// A machine may have reported before the last tier ended: the walk ends at whichever is later.
for i := len(out) - 1; i >= 0; i-- {
if out[i].at != nil {
if out[i].at.After(*end) {
e := *out[i].at
end = &e
}
break
}
}
said := ""
if len(w.Silent) > 0 {
sort.Strings(w.Silent)
said = "not waited for, not heard from: " + strings.Join(w.Silent, ", ")
}
return append(out, point{phase: PhaseApply, tier: -1, at: end, said: said})
}
type restSend struct {
module, node string
walkSent time.Time
what SentDeclaration
}
func askedAnyOf(p Plan, tier []string) bool {
for _, m := range tier {
if s := p.Modules[m]; s != nil && s.AskedAt != nil {
return true
}
}
return false
}
func earliest(a, b *time.Time) *time.Time {
if b == nil {
return a
}
if a == nil || b.Before(*a) {
t := *b
return &t
}
return a
}
func latest(a, b *time.Time) *time.Time {
if b == nil {
return a
}
if a == nil || b.After(*a) {
t := *b
return &t
}
return a
}
func appendOnce(to []string, s string) []string {
for _, x := range to {
if x == s {
return to
}
}
return append(to, s)
}
func orNot(s string) string {
if s == "" {
return "is not known"
}
return "(" + s + ")"
}
// words is a duration as a person reads it.
func words(d time.Duration) string {
if d < 0 {
return "-" + words(-d)
}
return d.Round(100 * time.Millisecond).String()
}
// FirstAppliedAfter is a machine's report of the send a walk made to it: the controller's `apply` durations, which
// measure every send to its first report (to-be 45 Phase 0), each row starting at the machine's own send time and
// naming the declaration it was about. send is what the walk kept of that send; walkSent its own record of the
// send to the rest, made after every machine of the rest was sent (novox/hq issue 582). FirstReportOf chooses.
func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool, error) {
from, to := applyRowsBetween(send, walkSent)
rows, err := i.store.Pool().Query(ctx,
`select kind, ref, started, took_ms, detail from duration
where subject = $2 and ((kind = $1 and started >= $3 and started < $4) or (kind = $5 and started >= $3))
order by started`,
DurationApply, node, from, to, DurationBusy)
if err != nil {
return AppliedReport{}, false, err
}
defer rows.Close()
var ds []Duration
for rows.Next() {
d := Duration{Subject: node, Node: node}
var ms int64
if err := rows.Scan(&d.Kind, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
return AppliedReport{}, false, err
}
d.Took = time.Duration(ms) * time.Millisecond
ds = append(ds, d)
}
if err := rows.Err(); err != nil {
return AppliedReport{}, false, err
}
r, ok := FirstReportOf(ds, send, walkSent)
return r, ok, nil
}
// applyRowsBetween is the span of a machine's `apply` durations that may hold its report of a walk's send.
func applyRowsBetween(send SentDeclaration, walkSent time.Time) (time.Time, time.Time) {
switch {
case !send.At.IsZero():
return send.At.Add(-time.Millisecond), send.At.Add(time.Millisecond)
case send.Digest != "":
return walkSent.Add(-restSendBound), walkSent.Add(ApplySilentAfter)
}
return walkSent.Add(-appliedSlack), walkSent.Add(ApplySilentAfter)
}
// FirstReportOf picks, from a machine's `apply` durations ordered by their start, its report of the send a walk made
// to it (novox/hq issue 582):
// - with the machine's own send time kept (every walk since issue 582): the duration of that very send, and of the
// declaration sent where the walk kept its digest. A report of a later send is never this walk's: a send made
// later, by another walk, would end this walk at that walk's send.
// - with only the declaration's digest kept (a walk kept before): the first duration about that declaration
// within restSendBound before the walk's record of its send, the machine sent first in the rest included.
// - with neither (a walk kept before ADR 0282's rest): the first duration started no earlier than appliedSlack
// before the walk's record of its send, as it was read before.
//
// A send past ApplySilentAfter is never read: its machine slept, and is listed silent (ADR 0282 decision 1).
//
// **A first report that said a container waits because it is busy** (novox/hq ADR 0335) is the machine's answer,
// and not yet the build running: its report is the end of its busy wait, the `busy-wait` duration of the same
// send, with the wait said apart (Busy). While none is kept the machine is still waiting (Waiting), never silent.
func FirstReportOf(ds []Duration, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
from, to := applyRowsBetween(send, walkSent)
for _, d := range ds {
if d.Kind == DurationBusy || d.Started.Before(from) || !d.Started.Before(to) {
continue
}
if send.Digest != "" && !strings.HasPrefix(d.Ref, send.Digest+"@") {
continue
}
answered := d.Started.Add(d.Took).UTC()
if d.Detail != ApplyBusy {
return AppliedReport{At: answered, Outcome: d.Detail}, true
}
for _, b := range ds {
if b.Kind == DurationBusy && b.Ref == d.Ref {
return AppliedReport{At: b.Started.Add(b.Took).UTC(), Outcome: OutcomeApplied, Answered: answered,
Busy: b.Took}, true
}
}
return AppliedReport{At: answered, Outcome: OutcomeApplied, Answered: answered, Waiting: true}, true
}
return AppliedReport{}, false
}
// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be, for a
// walk that kept neither the machine's send time nor what it sent.
const appliedSlack = 2 * time.Second
// restSendBound is how long before a walk's record of its send to the rest a machine of the rest may have been
// sent, for a walk that kept what it sent and not when: the send to the rest goes machine by machine, and took
// 19 to 28 seconds on 2026-10-12 (novox/hq issue 582).
const restSendBound = 5 * time.Minute
internal/inventory/walkphases_582_integration_test.go
-49
View File
@@ -1,49 +0,0 @@
//go:build integration
package inventory
import (
"testing"
"time"
)
func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
record, err := inv.AddNode(ctx, "g14")
if err != nil {
t.Fatal(err)
}
if _, err := inv.NextSequence(ctx, record.ID); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil {
t.Fatal(err)
}
sent, found, err := inv.SentTo(ctx, "g14")
if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" {
t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err)
}
if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok {
t.Fatalf("no report yet: %v %v", ok, err)
}
if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil {
t.Fatal(err)
}
// The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send.
r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second))
if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) {
t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err)
}
// A later send, reported: never this walk's.
if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil {
t.Fatal(err)
}
if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil {
t.Fatal(err)
}
again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second))
if !again.At.Equal(r.At) {
t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r)
}
}
internal/inventory/walkphases_582_test.go
-170
View File
@@ -1,170 +0,0 @@
package inventory
import (
"strings"
"testing"
"time"
)
// novox/hq issue 582: the walk's own record of its send to the rest is made after every machine of the rest was
// sent, and that send took 19 to 28 s on 2026-10-12. The machine sent first reported 5 s after its own send, long
// before the walk's record; read from the walk's record, its report was missed and the walk ended at another walk's
// send to it, minutes later. A walk of one tier and three machines of the rest, the send to the rest taking 20 s.
type sendRest582 struct {
judged, walkSent time.Time
walk Plan
rows map[string][]Duration
}
func walk582(t *testing.T) sendRest582 {
t.Helper()
judged := time.Date(2026, 10, 12, 17, 45, 35, 137000000, time.UTC)
ownSend := map[string]time.Duration{"g14": 500 * time.Millisecond, "novox": 10 * time.Second, "shanks": 20 * time.Second}
walkSent := judged.Add(20200 * time.Millisecond)
merged := judged.Add(-10 * time.Minute)
built, first := judged.Add(-3*time.Minute), judged.Add(-2*time.Minute)
rest := map[string]SentDeclaration{}
rows := map[string][]Duration{}
took := map[string]time.Duration{"g14": 5 * time.Second, "novox": 13 * time.Second, "shanks": 15 * time.Second}
for i, n := range []string{"g14", "novox", "shanks"} {
at := judged.Add(ownSend[n])
digest := "walk-" + n
rest[n] = SentDeclaration{Digest: digest, Sequence: int64(647 + i), At: at}
rows[n] = append(rows[n],
// This walk's send, and its report.
Duration{Kind: DurationApply, Subject: n, Node: n, Ref: digest + "@" + at.Format("2006-01-02T15:04:05.000000"),
Started: at, Took: took[n], Detail: OutcomeApplied},
// The next walk's send to the same machine, nine minutes later, and its report.
Duration{Kind: DurationApply, Subject: n, Node: n, Ref: "next-" + n + "@x",
Started: judged.Add(9*time.Minute + ownSend[n]), Took: 4 * time.Second, Detail: OutcomeApplied})
}
p := Plan{ID: "plan-582", State: PlanDone, Tier: 1, Tiers: [][]string{{"build-agent"}}, Created: merged,
Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/mesh-controller", Commit: "c", Merged: merged}}},
Times: &PlanTimes{Cut: &merged, Class: ClassCore},
Modules: map[string]*PlanModule{"build-agent": {State: "built", AskedAt: &merged, BuiltAt: &built,
FirstAt: &first, SentAt: &walkSent, Gate: &PlanGate{JudgedAt: &judged}, Rest: rest}}}
return sendRest582{judged: judged, walkSent: walkSent, walk: p, rows: rows}
}
// lookup answers from the rows as the controller's durations would.
func (w sendRest582) lookup(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
return FirstReportOf(w.rows[node], send, walkSent)
}
// The apply phase ends at the last machine's own report of this walk's send — shanks, sent at the end of the 20 s
// and reporting 15 s later — and never at the next walk's send; every machine's report is shown.
func TestTheApplyPhaseEndsAtEachMachinesOwnReport(t *testing.T) {
c := walk582(t)
w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
sumsUp(t, w)
want := c.judged.Add(35 * time.Second) // shanks: sent 20 s after the judgement, reported 15 s later
if !w.End.Equal(want) {
t.Fatalf("the walk ends at %s, want %s: its rest %+v", w.End, want, w.Rest)
}
if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != want.Sub(c.walkSent).Milliseconds() {
t.Fatalf("apply: %+v", got)
}
if len(w.Rest) != 3 || len(w.Silent) != 0 {
t.Fatalf("every machine of the rest is shown with its report: %+v, silent %v", w.Rest, w.Silent)
}
g14 := w.Rest[0]
if g14.Node != "g14" || g14.State != RestApplied || g14.Reported == nil ||
!g14.Reported.Equal(c.judged.Add(5500*time.Millisecond)) || g14.Took != "5s" || g14.Sequence != 647 ||
!g14.Sent.Equal(c.judged.Add(500*time.Millisecond)) || g14.SentSaid != "" {
t.Fatalf("g14, sent first and reporting 5 s after its own send: %+v", g14)
}
}
// A walk kept before its machines' own send times were: matched by the declaration it sent, within restSendBound
// of its record of the send, so the machine sent first is still found; a walk that kept neither is read as before.
func TestAnOlderWalkIsReadByWhatItSent(t *testing.T) {
c := walk582(t)
for n, s := range c.walk.Modules["build-agent"].Rest {
s.At = time.Time{}
c.walk.Modules["build-agent"].Rest[n] = s
}
w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
sumsUp(t, w)
if !w.End.Equal(c.judged.Add(35 * time.Second)) {
t.Fatalf("by the declaration sent: end %s, rest %+v", w.End, w.Rest)
}
if !strings.Contains(w.Rest[0].SentSaid, "walk's record") || !w.Rest[0].Sent.Equal(c.walkSent) {
t.Fatalf("a send time not kept is said: %+v", w.Rest[0])
}
// Nothing of the send kept: the first report no earlier than appliedSlack before the walk's record, as before.
if r, ok := FirstReportOf(c.rows["shanks"], SentDeclaration{}, c.walkSent); !ok || !r.At.Equal(c.judged.Add(35*time.Second)) {
t.Fatalf("a walk that kept nothing of its send: %+v %v", r, ok)
}
}
// A machine that said nothing of this walk's send is waiting, then silent past ApplySilentAfter: the report of a
// later send to it is never taken as this walk's, and never moves the walk's end.
func TestALaterSendsReportIsNeverThisWalks(t *testing.T) {
c := walk582(t)
c.rows["g14"] = c.rows["g14"][1:] // g14 slept through this walk's send; the next walk's send it reported
w := c.walk.WalkPhases(c.judged.Add(5*time.Minute), c.lookup)
if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") {
t.Fatalf("g14 not heard from yet: end %v, apply %+v", w.End, phase(w, PhaseApply, -1))
}
w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
sumsUp(t, w)
if len(w.Silent) != 1 || w.Silent[0] != "g14" || !w.End.Equal(c.judged.Add(35*time.Second)) {
t.Fatalf("g14 silent, the end at shanks' report: silent %v, end %s", w.Silent, w.End)
}
if w.Rest[0].State != RestSilent || w.Rest[0].Reported != nil {
t.Fatalf("g14's later report taken as this walk's: %+v", w.Rest[0])
}
}
// A machine sent by two tiers of one walk that said nothing of the first send has said it of the second, which
// carries the first tier's build too.
func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) {
c := walk582(t)
second := c.judged.Add(4 * time.Minute)
secondSent := second.Add(time.Second)
c.walk.Tiers = append(c.walk.Tiers, []string{"route-proxy"})
c.walk.Tier = 2
asked, built := c.judged.Add(time.Minute), c.judged.Add(2*time.Minute)
first := built.Add(time.Second)
c.walk.Modules["route-proxy"] = &PlanModule{State: "built", AskedAt: &asked, BuiltAt: &built, FirstAt: &first,
SentAt: &secondSent, Gate: &PlanGate{JudgedAt: &second},
Rest: map[string]SentDeclaration{"novox": {Digest: "walk-novox-2", Sequence: 1554, At: second.Add(500 * time.Millisecond)}}}
c.rows["novox"] = []Duration{{Kind: DurationApply, Subject: "novox", Ref: "walk-novox-2@x",
Started: second.Add(500 * time.Millisecond), Took: 14 * time.Second, Detail: OutcomeApplied}}
w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
sumsUp(t, w)
want := second.Add(14500 * time.Millisecond)
if !w.End.Equal(want) || len(w.Silent) != 0 {
t.Fatalf("end %s (want %s), silent %v, rest %+v", w.End, want, w.Silent, w.Rest)
}
for _, r := range w.Rest {
if r.Node == "novox" && (r.State != RestApplied || !r.Reported.Equal(want)) {
t.Fatalf("novox, heard by its later send of this walk: %+v", r)
}
}
}
// The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it —
// the send and the duration start at one moment, the node's sent_at.
// Issue 840 (ADR 0362): the walk's report of a machine is asked for with only the provisioned modules the walk sent
// there, never every module the machine's declaration hands to the provisioner.
func TestTheRestIsAskedAboutTheModulesTheWalkSent(t *testing.T) {
c := walk582(t)
rest := c.walk.Modules["build-agent"].Rest
for n, s := range rest {
s.Provisioned = map[string]string{"build-agent": "t1", "other": "t2"}
rest[n] = s
}
asked := 0
c.walk.WalkPhases(c.judged.Add(time.Hour), func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
asked++
if len(send.Provisioned) != 1 || send.Provisioned["build-agent"] != "t1" {
t.Fatalf("%s is asked about %v", node, send.Provisioned)
}
return c.lookup(node, send, walkSent)
})
if asked == 0 {
t.Fatal("no machine was asked")
}
}
internal/inventory/walkphases_integration_test.go
-77
View File
@@ -1,77 +0,0 @@
//go:build integration
package inventory
import (
"testing"
"time"
)
func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
p := walkOf(t, recordedWalk)
p.Revision, p.Epoch = 0, 0
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
back, err := inv.PlanByID(ctx, p.ID)
if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil ||
!back.Times.WindowClosed.Equal(*p.Times.WindowClosed) {
t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times)
}
if back.Delivery.Merges[0].Merged.IsZero() {
t.Fatal("a merge's time was not kept")
}
sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond)
if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok {
t.Fatalf("no report yet: %v %v", ok, err)
}
for _, d := range []Duration{
{Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied},
{Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied},
} {
if err := inv.RecordDuration(ctx, d); err != nil {
t.Fatal(err)
}
}
r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent)
if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) {
t.Fatalf("the report after the send: %+v %v %v", r, ok, err)
}
if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total {
t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err)
}
if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total",
Started: sent, Took: time.Minute}); err != nil {
t.Fatal(err)
}
if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total {
t.Fatalf("the total kept: %v %v %v", anyKept, total, err)
}
}
func TestALateReportIsSilentNotTheEnd(t *testing.T) {
sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
p := walkOf(t, recordedWalk)
p.Modules["mesh-delivery"].SentAt = &sent
p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}}
w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
if node == "server" {
return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true
}
return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true
})
if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) {
t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End)
}
inv := ForTest(t)
ctx := t.Context()
if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1",
Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil {
t.Fatal(err)
}
if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok {
t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err)
}
}
internal/inventory/walkphases_test.go
-263
View File
@@ -1,263 +0,0 @@
package inventory
import (
"encoding/json"
"strings"
"testing"
"time"
)
// A walk recorded on the live mesh on 2026-10-10 (mesh-delivery, one tier, one machine), as `delivery walks`
// gave it, with the moments ADR 0282 adds: its window closed ninety seconds after its merge was heard, and it was
// cut eleven seconds later.
const recordedWalk = `{
"id": "plan-1791654663505629616", "repository": "novox/mesh-catalog", "branch": "main",
"commit": "a14fa306f262d88bdcca83432a70df353d2eceb0", "merged": "2026-10-10T17:50:53Z",
"created": "2026-10-10T19:52:34.037755+02:00", "updated": "2026-10-10T19:55:37.974069+02:00",
"state": "done", "tier": 1, "tiers": [["mesh-delivery"]],
"modules": {"mesh-delivery": {"state": "built",
"asked_at": "2026-10-10T17:52:34.209423145Z", "built_at": "2026-10-10T17:52:50.818011314Z",
"sent_at": "2026-10-10T17:55:35.894985053Z", "first": ["novox"], "first_at": "2026-10-10T17:53:07.287136827Z",
"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3,
"verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}}},
"delivery": {"awaits": "", "merges": [{"repository": "novox/mesh-catalog",
"commit": "a14fa306f262d88bdcca83432a70df353d2eceb0", "number": 187, "merged": "2026-10-10T17:50:53Z"}]},
"times": {"window_closed": "2026-10-10T17:52:23Z", "cut": "2026-10-10T17:52:34.037755Z", "class": "core"}
}`
func walkOf(t *testing.T, raw string) Plan {
t.Helper()
var p Plan
if err := json.Unmarshal([]byte(raw), &p); err != nil {
t.Fatal(err)
}
return p
}
// sumsUp checks the phases' measured time and the unknown time add up to the total, to the millisecond.
func sumsUp(t *testing.T, w *WalkPhases) {
t.Helper()
if w.End == nil || w.From == nil {
t.Fatalf("the walk has no end: %+v", w)
}
var sum int64
for _, ph := range w.Phases {
if ph.State == PhaseMeasured {
if ph.Start == nil || ph.End == nil || ph.End.Sub(*ph.Start).Milliseconds() != ph.TookMS {
t.Errorf("phase %s %d says %dms between %v and %v", ph.Name, ph.Tier, ph.TookMS, ph.Start, ph.End)
}
sum += ph.TookMS
}
if ph.State == PhaseUnknown && ph.TookMS != 0 {
t.Errorf("an unknown phase %s says a time: %dms", ph.Name, ph.TookMS)
}
}
if sum+w.UnknownMS != w.TotalMS || w.End.Sub(*w.From).Milliseconds() != w.TotalMS {
t.Fatalf("the phases add up to %dms and %dms unknown, the total is %dms (%s): %+v", sum, w.UnknownMS,
w.TotalMS, w.End.Sub(*w.From), w.Phases)
}
}
func phase(w *WalkPhases, name string, tier int) WalkPhase {
for _, ph := range w.Phases {
if ph.Name == name && ph.Tier == tier {
return ph
}
}
return WalkPhase{}
}
// The phases of a recorded walk add up to its measured total: its merge at 17:50:53 to its verdict on its only
// machine at 17:55:35.894, 4m42.894s.
func TestARecordedWalksPhasesAddUpToItsTotal(t *testing.T) {
w := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
sumsUp(t, w)
if w.TotalMS != 282894 || w.Class != ClassCore || w.UnknownMS != 0 {
t.Fatalf("total %dms (want 282894), class %q, unknown %d", w.TotalMS, w.Class, w.UnknownMS)
}
for name, want := range map[string]int64{PhaseWindow: 90000, PhaseQueued: 11037, PhaseBuild: 16781,
PhaseSend: 16469, PhaseJudge: 148607, PhaseRest: 0} {
tier := 0
if name == PhaseWindow || name == PhaseQueued {
tier = -1
}
if got := phase(w, name, tier); got.State != PhaseMeasured || got.TookMS != want {
t.Errorf("%s: %s %dms, want measured %dms", name, got.State, got.TookMS, want)
}
}
// The cut to the first ask (171ms) is a time no phase of ADR 0282's table names: it is counted in the build.
if got := phase(w, PhaseWord, -1); got.State != PhaseNone {
t.Errorf("a walk that waits for no word says its word phase %s", got.State)
}
if got := phase(w, PhaseApply, -1); got.State != PhaseNone || !strings.Contains(got.Said, "no machine beyond the first: its one machine's report was its run") {
t.Errorf("one machine only: apply %s (%s)", got.State, got.Said)
}
}
// Two tiers, a machine of the rest each, both reports read: every phase measured, the walk ends at the last
// report applied, and the phases add up.
func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
at := func(s string) *time.Time {
v, err := time.Parse(time.RFC3339Nano, "2026-10-10T18:"+s+"Z")
if err != nil {
t.Fatal(err)
}
return &v
}
p := Plan{ID: "plan-2", State: PlanDone, Tier: 2, Tiers: [][]string{{"a"}, {"b"}}, Created: *at("01:40"),
Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/x", Commit: "c1", Merged: *at("00:00")},
{Repository: "novox/x", Commit: "c0", Merged: *at("00:30")}}},
Times: &PlanTimes{WindowClosed: at("01:30"), Cut: at("01:40"), Class: ClassLeaf},
Modules: map[string]*PlanModule{
"a": {State: "built", AskedAt: at("01:41"), BuiltAt: at("02:05"), FirstAt: at("02:20"), SentAt: at("05:00"),
Gate: &PlanGate{JudgedAt: at("04:50")}, Rest: map[string]SentDeclaration{"ace": {Digest: "d1"}}},
"b": {State: "built", AskedAt: at("05:10"), BuiltAt: at("05:40"), FirstAt: at("05:50"), SentAt: at("08:00.5"),
Gate: &PlanGate{JudgedAt: at("07:59")}, Rest: map[string]SentDeclaration{"shanks": {Digest: "d2"}}},
}}
reports := map[string]AppliedReport{"ace": {At: *at("05:12"), Outcome: OutcomeApplied},
"shanks": {At: *at("08:15.25"), Outcome: OutcomeApplied}}
w := p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
r, ok := reports[node]
return r, ok
})
sumsUp(t, w)
if w.TotalMS != (8*time.Minute + 15250*time.Millisecond).Milliseconds() {
t.Fatalf("the walk's delivery time runs from its earliest merge to the last report: %s", w.Total)
}
if got := phase(w, PhaseBetween, 1); got.TookMS != 10000 {
t.Errorf("between the tiers: %dms", got.TookMS)
}
if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != 14750 {
t.Errorf("apply: %s %dms", got.State, got.TookMS)
}
if len(w.Merges) != 2 {
t.Errorf("each merge's start is said, for its own delivery time: %+v", w.Merges)
}
// A report not yet in: the walk has no end, and its apply is unknown — never zero.
delete(reports, "shanks")
w = p.WalkPhases(*at("10:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
r, ok := reports[node]
return r, ok
})
if w.End != nil || w.TotalMS != 0 {
t.Fatalf("a walk whose rest has not reported has no end: %+v", w)
}
if got := phase(w, PhaseApply, -1); got.State != PhaseUnknown || got.TookMS != 0 || !strings.Contains(got.Said, "shanks") {
t.Errorf("apply while shanks has not reported: %+v", got)
}
// Silent past the bound: listed, not waited for.
w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
r, ok := reports[node]
return r, ok
})
sumsUp(t, w)
if len(w.Silent) != 1 || w.Silent[0] != "shanks" {
t.Errorf("a machine silent past the bound is said, not waited for: %+v", w.Silent)
}
// A failed report: the walk has no end, said.
reports["shanks"] = AppliedReport{At: *at("08:10"), Outcome: OutcomeFailed}
w = p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
r, ok := reports[node]
return r, ok
})
if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "shanks (failed)") {
t.Errorf("a failed apply ends nothing: %+v", phase(w, PhaseApply, -1))
}
}
// A moment that is missing makes the phases around it unknown, and their span unknown time: never a zero.
func TestAMissingMomentIsUnknownNeverZero(t *testing.T) {
p := walkOf(t, recordedWalk)
p.Modules["mesh-delivery"].BuiltAt = nil
w := p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
sumsUp(t, w)
if got := phase(w, PhaseBuild, 0); got.State != PhaseUnknown || got.TookMS != 0 {
t.Errorf("a build without its moment: %+v", got)
}
if got := phase(w, PhaseSend, 0); got.State != PhaseUnknown {
t.Errorf("the send after a build without its moment: %+v", got)
}
if w.UnknownMS != 16781+16469 {
t.Errorf("the unknown time is the span between the moments around it: %dms", w.UnknownMS)
}
// A walk kept before ADR 0282: its window and its wait together, and its apply unknown.
p = walkOf(t, recordedWalk)
p.Times = nil
w = p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
if got := phase(w, PhaseWindow, -1); got.State != PhaseUnknown {
t.Errorf("a window not kept: %+v", got)
}
if got := phase(w, PhaseApply, -1); got.State != PhaseUnknown || w.End != nil {
t.Errorf("a rest not kept: %+v, end %v", got, w.End)
}
}
// An open walk is said open since its last moment, with no end.
func TestAnOpenWalkHasNoEnd(t *testing.T) {
p := walkOf(t, recordedWalk)
p.State, p.Tier = PlanRolling, 0
p.Modules["mesh-delivery"].SentAt = nil
p.Modules["mesh-delivery"].Gate.JudgedAt = nil
w := p.WalkPhases(time.Date(2026, 10, 10, 17, 54, 0, 0, time.UTC), nil)
if w.End != nil || !strings.HasPrefix(w.Said, "its end is unknown") && !strings.HasPrefix(w.Said, "open") {
t.Fatalf("an open walk: %+v", w)
}
if got := phase(w, PhaseBuild, 0); got.State != PhaseMeasured {
t.Errorf("an open walk's phases so far are measured: %+v", got)
}
}
// A gate keeps every pass and the first reading after one that did not pass, at most maxReadings.
func TestAGateKeepsItsReadings(t *testing.T) {
var g PlanGate
t0 := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC)
g.Read(t0, false, "starting")
g.Read(t0.Add(time.Second), false, "starting")
g.Read(t0.Add(40*time.Second), true, "")
g.Read(t0.Add(80*time.Second), true, "")
g.Read(t0.Add(81*time.Second), false, "gone")
g.Read(t0.Add(82*time.Second), false, "gone")
if len(g.Readings) != 4 || g.Readings[0].Said != "starting" || !g.Readings[2].Healthy || g.Readings[3].Said != "gone" {
t.Fatalf("readings: %+v", g.Readings)
}
for i := 0; i < 50; i++ {
g.Read(t0.Add(time.Duration(100+i)*time.Second), true, "")
}
if len(g.Readings) != maxReadings {
t.Fatalf("readings are bounded: %d", len(g.Readings))
}
}
// A walk's moments are kept and read back with it; a machine's first report after a send is read from the
// controller's apply durations.
// A machine that slept past the bound and reported later is listed silent: its late report never moves the
// walk's end; and a report sent past the bound is not read as the walk's.
// Moments out of order — #206's own walk on 2026-10-10 kept its first send 783 ms before its build — are said
// unknown, never a negative phase: the phase before is unknown and its time joins the unknown time, the phase
// out of order has no time, and the phases still add up to the total.
func TestMomentsOutOfOrderAreUnknownNeverNegative(t *testing.T) {
p := walkOf(t, recordedWalk)
built := time.Date(2026, 10, 10, 17, 53, 8, 70000000, time.UTC) // 783 ms after its first send
p.Modules["mesh-delivery"].BuiltAt = &built
w := p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
sumsUp(t, w)
for _, ph := range w.Phases {
if ph.TookMS < 0 {
t.Fatalf("a negative phase: %+v", ph)
}
}
build, send := phase(w, PhaseBuild, 0), phase(w, PhaseSend, 0)
if build.State != PhaseUnknown || send.State != PhaseUnknown || send.TookMS != 0 ||
!strings.Contains(send.Said, "out of order") || !strings.Contains(build.Said, "out of order") {
t.Fatalf("build %+v, send-first %+v", build, send)
}
// The build's span (cut to its later moment) is unknown time; the judgement is measured from the later moment.
if w.UnknownMS != built.Sub(time.Date(2026, 10, 10, 17, 52, 34, 37000000, time.UTC)).Milliseconds() {
t.Fatalf("unknown %dms", w.UnknownMS)
}
if j := phase(w, PhaseJudge, 0); j.State != PhaseMeasured || j.Start == nil || !j.Start.Equal(built) {
t.Fatalf("the judgement is measured from the later moment: %+v", j)
}
}
internal/inventory/walksaid.go
-81
View File
@@ -1,81 +0,0 @@
package inventory
import "time"
// A walk says when it starts and when it ends (novox/hq ADR 0308 decision 1): the save that starts it and the
// save that ends it are marked, so `plan-moved` names what the walk touches, its bound and its outcome on those
// two saves, and on no other. The marks are kept in the walk's times, so a controller replaced mid-walk (a core
// walk replaces it) neither says a start again nor misses the end.
// The moments of a walk `plan-moved` names.
const (
WalkStarted = "started"
WalkEnded = "ended"
)
// The outcomes of an ended walk (ADR 0308 decision 1).
const (
WalkDelivered = "delivered"
WalkFailed = "failed"
WalkStopped = "stopped"
)
// Live says the walk is walking: cut, not waiting for its delivery's word, and not ended.
func (p Plan) Live() bool { return p.Open() && !p.Batch() && !p.Waiting() }
// Ended says the walk is in a state it does not leave by itself: done, failed or superseded.
func (p Plan) Ended() bool {
return p.State == PlanDone || p.State == PlanFailed || p.State == PlanSuperseded
}
// Outcome is how an ended walk ended: delivered when done; stopped when a person or the delivery's owner stopped
// it, or a newer walk took it over; failed otherwise. Empty for a walk not ended.
func (p Plan) Outcome() string {
switch {
case p.State == PlanDone:
return WalkDelivered
case p.State == PlanSuperseded:
return WalkStopped
case p.State == PlanFailed && p.Delivery != nil && p.Delivery.Stopped != "":
return WalkStopped
case p.State == PlanFailed:
return WalkFailed
}
return ""
}
// markWalk marks the save that starts or ends the walk, in its times and in WalkSaid, and returns what undoes
// the mark should the save not be written. A walk starts on the first save that finds it live — when it is cut
// and needs no word, or when it is let go — which is before its first tier is asked, and so before anything is
// sent: what it restarts is held from its first send on. It starts again when a failed walk is retried. It ends
// on the first save that finds it ended after it started; one that ends without ever starting (a batch that
// moved nothing, a walk stopped while it waited for its word) touched nothing and says neither. A walk kept
// before its times were (ADR 0282 decision 6) is not marked; a release plan is, its times holding only these.
func (p *Plan) markWalk(now time.Time) (undo func()) {
p.WalkSaid = ""
if p.Batch() || (p.Times == nil && p.Release == nil) {
return func() {}
}
var was *PlanTimes
if p.Times != nil {
kept := *p.Times
was = &kept
}
times := PlanTimes{}
if p.Times != nil {
times = *p.Times
}
at := now.UTC()
switch {
case p.Live() && (times.Started == nil || times.Ended != nil):
times.Started, times.Ended = &at, nil
p.WalkSaid = WalkStarted
case p.Ended() && times.Started != nil && times.Ended == nil:
times.Ended = &at
p.WalkSaid = WalkEnded
default:
return func() {}
}
p.Times = &times
return func() { p.Times, p.WalkSaid = was, "" }
}
internal/inventory/walksaid_integration_test.go
-50
View File
@@ -1,50 +0,0 @@
//go:build integration
package inventory
import (
"testing"
"time"
)
func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
cut := time.Now().UTC().Add(-time.Minute)
p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding,
Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}},
Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}}
var said []string
was := PlanSaved
PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) }
t.Cleanup(func() { PlanSaved = was })
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
back, err := inv.PlanByID(ctx, p.ID)
if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf {
t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times)
}
// Read back by another controller and saved: not said again.
back.State = PlanRolling
if err := inv.SavePlan(ctx, &back); err != nil {
t.Fatal(err)
}
// A stale copy ending it is refused, and unmarked.
stale := p
stale.Revision = 1
stale.State = PlanDone
if err := inv.SavePlan(ctx, &stale); err == nil {
t.Fatal("a stale save is refused")
}
if stale.WalkSaid != "" || stale.Times.Ended != nil {
t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times)
}
back.State = PlanDone
if err := inv.SavePlan(ctx, &back); err != nil {
t.Fatal(err)
}
if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded {
t.Fatalf("said: %q, want started, nothing, ended", said)
}
}
internal/inventory/walksaid_test.go
-89
View File
@@ -1,89 +0,0 @@
package inventory
import (
"testing"
"time"
)
// A walk is said started on the save that finds it live, ended on the save that finds it ended after it started,
// and neither on any other save (novox/hq ADR 0308 decision 1).
func TestAWalkIsSaidStartedOnceAndEndedOnce(t *testing.T) {
now := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC)
cut := now.Add(-time.Second)
p := Plan{ID: "plan-1", State: PlanBuilding, Tiers: [][]string{{"messenger"}},
Modules: map[string]*PlanModule{"messenger": {}}, Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}}
p.markWalk(now)
if p.WalkSaid != WalkStarted || p.Times.Started == nil || !p.Times.Started.Equal(now) {
t.Fatalf("a live walk's first save starts it: %q %+v", p.WalkSaid, p.Times)
}
p.State = PlanRolling
p.markWalk(now.Add(time.Minute))
if p.WalkSaid != "" || !p.Times.Started.Equal(now) {
t.Fatalf("a later save says nothing and keeps the start: %q %+v", p.WalkSaid, p.Times)
}
p.State = PlanDone
p.markWalk(now.Add(2 * time.Minute))
if p.WalkSaid != WalkEnded || p.Times.Ended == nil || p.Outcome() != WalkDelivered {
t.Fatalf("the save that finds it done ends it, delivered: %q %+v %q", p.WalkSaid, p.Times, p.Outcome())
}
p.markWalk(now.Add(3 * time.Minute))
if p.WalkSaid != "" {
t.Fatalf("an ended walk saved again says nothing: %q", p.WalkSaid)
}
// Retried, it walks again: a new start.
p.State = PlanBuilding
p.markWalk(now.Add(4 * time.Minute))
if p.WalkSaid != WalkStarted || p.Times.Ended != nil || !p.Times.Started.Equal(now.Add(4*time.Minute)) {
t.Fatalf("a retried walk starts again: %q %+v", p.WalkSaid, p.Times)
}
}
// What is never said: a batch, a walk waiting for its word, a walk that ends without having started, and a walk
// kept before its times were. A release plan is said.
func TestWhatIsNotAWalkStartIsNotSaid(t *testing.T) {
now := time.Now()
for name, p := range map[string]Plan{
"a batch": {State: PlanAssembling, Times: &PlanTimes{}},
"waiting for a word": {State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery"}},
"ended, never begun": {State: PlanDone, Times: &PlanTimes{}},
"kept before times": {State: PlanBuilding},
"stopped while waits": {State: PlanFailed, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Stopped: "x"}},
} {
p.markWalk(now)
if p.WalkSaid != "" {
t.Errorf("%s: said %q", name, p.WalkSaid)
}
}
r := Plan{State: PlanRolling, Release: &PlanRelease{Order: []string{"ace"}}}
r.markWalk(now)
if r.WalkSaid != WalkStarted || r.Times == nil || r.Times.Started == nil || r.Times.Class != "" {
t.Fatalf("a release plan is said started, its times holding only that: %q %+v", r.WalkSaid, r.Times)
}
// Let go, the walk starts.
go1 := now
w := Plan{State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Go: &go1}}
w.markWalk(now)
if w.WalkSaid != WalkStarted {
t.Fatalf("a walk let go starts: %q", w.WalkSaid)
}
}
// An ended walk's outcome: delivered, failed, or stopped by a person or the delivery's owner, or by a newer walk.
func TestAWalksOutcome(t *testing.T) {
for want, p := range map[string]Plan{
WalkDelivered: {State: PlanDone},
WalkFailed: {State: PlanFailed},
WalkStopped: {State: PlanFailed, Delivery: &PlanDelivery{Stopped: "mesh-delivery"}},
"": {State: PlanRolling},
} {
if got := p.Outcome(); got != want {
t.Errorf("%s: outcome %q, want %q", p.State, got, want)
}
}
if (Plan{State: PlanSuperseded}).Outcome() != WalkStopped {
t.Error("a superseded walk is stopped")
}
}
// The marks are kept with the walk, so a controller replaced mid-walk neither says its start again nor misses
// its end; and a save refused leaves the walk unmarked, so the save that is made says it.
internal/link/enrolment.go
-32
View File
@@ -449,17 +449,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil {
return false, err
}
// And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that
// orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one.
if report.Ordered() {
if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil {
return false, err
}
// And whether it reads a module per resource and a build per module (novox/hq issue 1145), the same way.
if err := e.Inventory.RecordReadsModules(ctx, node.ID, report.ReadsModules); err != nil {
return false, err
}
}
if report.Ordered() {
account := AccountOf(report)
news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing,
@@ -496,27 +485,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
return false, err
}
}
// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
//
// **A wait for a busy container is its own measure** (novox/hq ADR 0335): the first report that says one
// waits is the machine's answer, kept as its apply with the detail ApplyBusy; the report that says the
// container replaced ends the wait, kept as a `busy-wait` of the same send — so the apply's time holds no
// busy time, and the walk reads the machine as answered and waiting, never as silent.
outcome := doing.Outcome
if doing.Waiting() {
outcome = inventory.ApplyBusy
log.Printf("%s applied declaration %s and leaves a busy container running until it says idle: %s",
report.Node, short(report.Declared), busyWords(doing.Busy))
} else if replacedAfterWaiting(doing.Busy) {
if err := e.Inventory.RecordBusyDuration(ctx, report.Node, report.Declared); err != nil {
log.Printf("how long %s waited for a busy container could not be recorded: %v", report.Node, err)
}
}
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, outcome); err != nil {
log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
}
// A refusal, a failure, or a bare word that the node is there — none of them is an account of
// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
// as though it were the whole would tell a rebuilding node to remove what it still has.