Compare commits
58
Commits
407dc86a74
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b8c810a349 | ||
|
|
0f21fe7888 | ||
|
|
bd257afbd5 | ||
|
|
357e08a4f6 | ||
|
|
a83b921af0 | ||
|
|
3d09d5235f | ||
|
|
f883830975 | ||
|
|
96325c295c | ||
|
|
9ca2c2bfb3 | ||
|
|
20c8d9454a | ||
|
|
3d192a4857 | ||
|
|
8af77c2c75 | ||
|
|
fd522f4322 | ||
|
|
7d80155d16 | ||
|
|
bafc31bf85 | ||
|
|
fbf5a3f909 | ||
|
|
20e14872ef | ||
|
|
5702187695 | ||
|
|
a79182e53b | ||
|
|
ff16938096 | ||
|
|
e3f22c737f | ||
|
|
2b99d455fc | ||
|
|
9e99f31c23 | ||
|
|
df39a499ef | ||
|
|
11725908ec | ||
|
|
379a52c2e8 | ||
|
|
b0d489a86c | ||
|
|
6355176fda | ||
|
|
300d3c1bf4 | ||
|
|
baf1501d81 | ||
|
|
a5ba47e23c | ||
|
|
6a62a7e552 | ||
|
|
5890eb104d | ||
|
|
262b39ae09 | ||
|
|
2f2a8bb57e | ||
|
|
1cdf12df00 | ||
|
|
f614eeb445 | ||
|
|
53a3c91f53 | ||
|
|
df5f01bfa1 | ||
|
|
32014ac6de | ||
|
|
c7cb9f0a2c | ||
|
|
87410d79ed | ||
|
|
1c3c3edc7d | ||
|
|
b2e2bf0638 | ||
|
|
e231185572 | ||
|
|
b12755c163 | ||
|
|
abf37c7212 | ||
|
|
ead062569e | ||
|
|
f4ddddd59d | ||
|
|
f45b4cb213 | ||
|
|
210f7cf5e6 | ||
|
|
eea5866216 | ||
|
|
b5da0b2161 | ||
|
|
9cb08299b3 | ||
|
|
5fa523f608 | ||
|
|
9a5c02bd60 | ||
|
|
9ce77e4e7b | ||
|
|
470c97654b |
@@ -27,6 +27,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -290,6 +291,26 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
if err := work.Began(ctx); err != nil {
|
if err := work.Began(ctx); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||||
}
|
}
|
||||||
|
// **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk
|
||||||
|
// commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge
|
||||||
|
// (issue 726). A group's check, a try and a build watch nothing.
|
||||||
|
var superseded atomic.Bool
|
||||||
|
if w, ok := work.(interface {
|
||||||
|
WatchPull(context.Context, func(link.PullKey)) (func(), error)
|
||||||
|
}); ok && request.Check != nil {
|
||||||
|
stopWatching, err := w.WatchPull(building, func(k link.PullKey) {
|
||||||
|
superseded.Store(true)
|
||||||
|
say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped",
|
||||||
|
k.Head, k.Trunk, request.Ref, request.Check.Trunk))
|
||||||
|
cancel()
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n",
|
||||||
|
request.Check.Owner, request.Check.Repo, request.Check.Number, err)
|
||||||
|
} else {
|
||||||
|
defer stopWatching()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
result := link.BuildResult{
|
result := link.BuildResult{
|
||||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||||
@@ -382,7 +403,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
if mine != nil {
|
if mine != nil {
|
||||||
killed = h.returned(mine) && err != nil
|
killed = h.returned(mine) && err != nil
|
||||||
}
|
}
|
||||||
if killed {
|
if superseded.Load() && err != nil {
|
||||||
|
// Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is.
|
||||||
|
result.Failed = link.SupersededByKey
|
||||||
|
say("failed", link.SupersededByKey)
|
||||||
|
} else if killed {
|
||||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||||
// error it ended with is the kill's consequence, not a fault of the source.
|
// error it ended with is the kill's consequence, not a fault of the source.
|
||||||
result.Failed = link.KilledByHand
|
result.Failed = link.KilledByHand
|
||||||
|
|||||||
@@ -391,10 +391,10 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
// says what it did in one line. Nothing for a module that declares no broker secret and runs no
|
||||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
// provisioner client; nothing for one whose user is already minted (a credential is rotated on
|
||||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the
|
||||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
@@ -406,6 +406,10 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||||
|
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||||
|
// A module running the provisioner client alone is minted the client's account (issue 917).
|
||||||
|
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
||||||
|
}
|
||||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|||||||
+139
-72
@@ -27,9 +27,10 @@ import (
|
|||||||
// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier
|
// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier
|
||||||
// ones. The walk names every merge it answers.
|
// ones. The walk names every merge it answers.
|
||||||
//
|
//
|
||||||
// - One walk at a time. Merges that come while a walk runs join the next batch. A walk that started is never
|
// - One walk at a time beside a related one (ADR 0407 decision 4, issue 973). Merges that come while a walk
|
||||||
// superseded; one still waiting for its delivery's word is folded into the next batch's walk, which
|
// runs join the next batch; a closed batch that shares nothing with an open walk is cut beside it. A walk
|
||||||
// answers every merge it answered — the only supersession left.
|
// that started is never superseded; one still waiting for its delivery's word is folded into the walk of a
|
||||||
|
// batch related to it, which answers every merge it answered — the only supersession left.
|
||||||
// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch
|
// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch
|
||||||
// still assembling, an open walk that builds everything it moves, or a walk done that did, which then
|
// still assembling, an open walk that builds everything it moves, or a walk done that did, which then
|
||||||
// names it at once. Otherwise it joins the next batch.
|
// names it at once. Otherwise it joins the next batch.
|
||||||
@@ -39,8 +40,9 @@ import (
|
|||||||
//
|
//
|
||||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
||||||
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
||||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Walks stay one at
|
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Such a batch is cut
|
||||||
// a time: such a batch is cut when no walk is open, and queued behind the one that is otherwise.
|
// at once, beside any open walk it shares nothing with, and queued behind the first one it is related to
|
||||||
|
// (cutUnrelated).
|
||||||
//
|
//
|
||||||
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
||||||
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
||||||
@@ -61,13 +63,14 @@ var (
|
|||||||
// mergeWindowFileVar names the file the controller's window settings are written to.
|
// mergeWindowFileVar names the file the controller's window settings are written to.
|
||||||
const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE"
|
const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE"
|
||||||
|
|
||||||
// batchCutEvery is how often the controller looks whether a batch's window closed.
|
|
||||||
const batchCutEvery = 5 * time.Second
|
|
||||||
|
|
||||||
// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before
|
// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before
|
||||||
// the controller is said to have failed to cut it (ADR 0276 decision 9).
|
// the controller is said to have failed to cut it (ADR 0276 decision 9).
|
||||||
const batchLateAfter = time.Minute
|
const batchLateAfter = time.Minute
|
||||||
|
|
||||||
|
// mergeGrace is how long before a batch's first merge a merge may have been made and still be told apart from one
|
||||||
|
// made before the controller began keeping merges (owedLate).
|
||||||
|
const mergeGrace = 10 * time.Minute
|
||||||
|
|
||||||
// edgeGroupOrder is a delivery group's order made an edge of a walk's tiers (ADR 0276 decision 5): the
|
// edgeGroupOrder is a delivery group's order made an edge of a walk's tiers (ADR 0276 decision 5): the
|
||||||
// modules of the member after depend on the modules of the member before, and are gated on them.
|
// modules of the member after depend on the modules of the member before, and are gated on them.
|
||||||
const edgeGroupOrder = "group-order"
|
const edgeGroupOrder = "group-order"
|
||||||
@@ -125,8 +128,7 @@ func durationSetting(v any) (time.Duration, bool) {
|
|||||||
// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard,
|
// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard,
|
||||||
// and cuts what is due.
|
// and cuts what is due.
|
||||||
func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error {
|
func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error {
|
||||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
// One merge acted on at a time, whoever hands it over (novox/hq issue 266).
|
||||||
// what the bus did not hand over (novox/hq issue 266).
|
|
||||||
actingOnMerges.Lock()
|
actingOnMerges.Lock()
|
||||||
defer actingOnMerges.Unlock()
|
defer actingOnMerges.Unlock()
|
||||||
|
|
||||||
@@ -245,8 +247,8 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
|||||||
}
|
}
|
||||||
|
|
||||||
// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut
|
// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut
|
||||||
// marks every module it moved seen at that moment, so a merge of that branch the bus lost and the catch-up
|
// marks every module it moved seen at that moment, so a merge of that branch
|
||||||
// hands over later — made before the cut, heard after it — reads as history and was dropped. One that touches
|
// handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches
|
||||||
// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them,
|
// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them,
|
||||||
// is kept; one made before it began is old news, as before.
|
// is kept; one made before it began is old news, as before.
|
||||||
func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time,
|
func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time,
|
||||||
@@ -340,8 +342,8 @@ func ownPath(touches []inventory.Entry) bool {
|
|||||||
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
||||||
// why.
|
// why.
|
||||||
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
||||||
// once means only that: walks stay one at a time (ADR 0276 decision 4), so with a walk open the batch is
|
// once means only that: with a walk open that it is related to the batch is queued behind it as any closed
|
||||||
// queued behind it as any closed batch is.
|
// batch is (ADR 0276 decision 4, ADR 0407 decision 4); beside one it shares nothing with it is cut.
|
||||||
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
||||||
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
||||||
edges, err := inv.Dependencies(ctx)
|
edges, err := inv.Dependencies(ctx)
|
||||||
@@ -406,7 +408,9 @@ func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inve
|
|||||||
}
|
}
|
||||||
carry := newFootprint()
|
carry := newFootprint()
|
||||||
if !own {
|
if !own {
|
||||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch may fold every one.
|
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch folds those it is related to
|
||||||
|
// (cutUnrelated). Every one is counted here, which only makes a merge join a batch more readily: it waits
|
||||||
|
// for the window, never walks beside what it shares something with (issue 973).
|
||||||
plans, err := inv.OpenPlans(ctx)
|
plans, err := inv.OpenPlans(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
@@ -544,6 +548,11 @@ func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.Bat
|
|||||||
// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they
|
// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they
|
||||||
// answer, its window, and assembling or queued behind a walk. Saved only when that changed.
|
// answer, its window, and assembling or queued behind a walk. Saved only when that changed.
|
||||||
func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error {
|
func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error {
|
||||||
|
return keepBatchBecause(ctx, inv, b, now, behind, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepBatchBecause is keepBatch for a batch queued behind a walk it is related to, which its words say (issue 973).
|
||||||
|
func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind, why string) error {
|
||||||
merges, err := inv.MergesOf(ctx, b.ID)
|
merges, err := inv.MergesOf(ctx, b.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -574,19 +583,25 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
|||||||
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
||||||
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
||||||
}
|
}
|
||||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(),
|
||||||
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
||||||
b.State = inventory.PlanAssembling
|
b.State = inventory.PlanAssembling
|
||||||
if behind != "" && windowClosed(*b, now) {
|
if behind != "" && windowClosed(*b, now) {
|
||||||
b.State = inventory.PlanQueued
|
b.State = inventory.PlanQueued
|
||||||
} else {
|
} else {
|
||||||
b.Delivery.Batch.Behind = ""
|
b.Delivery.Batch.Behind, b.Delivery.Batch.BehindWhy = "", ""
|
||||||
}
|
}
|
||||||
b.Note = batchWords(*b, now)
|
b.Note = batchWords(*b, now)
|
||||||
if unNoted(*b) == before {
|
if unNoted(*b) == before {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return inv.SavePlan(ctx, b)
|
if err := inv.SavePlan(ctx, b); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The window's clock follows the record, only when the record moved: a merge made the quiet key again, a
|
||||||
|
// batch that opened made its most (ADR 0407 decision 1).
|
||||||
|
armWindow(ctx, *b, now)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last
|
// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last
|
||||||
@@ -739,31 +754,10 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
|
|||||||
return inv.SavePlan(ctx, p)
|
return inv.SavePlan(ctx, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
// batchCutter cuts what is due on a timer, until the context ends: a window closes with no merge to say so.
|
|
||||||
func batchCutter(ctx context.Context, open *stores) {
|
|
||||||
tick := time.NewTicker(batchCutEvery)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
}
|
|
||||||
release, err := open.inventory.HoldPlans(ctx, false)
|
|
||||||
if err != nil {
|
|
||||||
continue // whoever holds the plans moves them, and cuts at the end
|
|
||||||
}
|
|
||||||
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
|
||||||
fmt.Printf("batches: %v\n", err)
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search
|
// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search
|
||||||
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone, or a
|
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone; else
|
||||||
// closed batch cut into one walk with the waiting walks folded in; with a walk open, a closed batch said
|
// each closed batch cut into a walk with the waiting walks it is related to folded in, unless it is related to a
|
||||||
// queued behind it.
|
// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits.
|
||||||
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
if err := searchAfterFailures(ctx, inv); err != nil {
|
if err := searchAfterFailures(ctx, inv); err != nil {
|
||||||
@@ -780,8 +774,7 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var started *inventory.Plan
|
var started, waiting []inventory.Plan
|
||||||
var waiting []inventory.Plan
|
|
||||||
for i := range plans {
|
for i := range plans {
|
||||||
p := plans[i]
|
p := plans[i]
|
||||||
if p.Release != nil {
|
if p.Release != nil {
|
||||||
@@ -789,8 +782,8 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
|||||||
}
|
}
|
||||||
if p.Waiting() {
|
if p.Waiting() {
|
||||||
waiting = append(waiting, p)
|
waiting = append(waiting, p)
|
||||||
} else if started == nil {
|
} else {
|
||||||
started = &plans[i]
|
started = append(started, p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
batches, err := inv.Batches(ctx)
|
batches, err := inv.Batches(ctx)
|
||||||
@@ -805,46 +798,116 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if started != nil {
|
|
||||||
// **One walk at a time** (ADR 0276 decision 4): every batch waits behind it, whatever its window says.
|
|
||||||
return keepAll(started.ID)
|
|
||||||
}
|
|
||||||
if len(alone) > 0 {
|
if len(alone) > 0 {
|
||||||
|
// A failed walk's search walks its merges alone, one at a time: everything waits for it (kept as it was,
|
||||||
|
// issue 973).
|
||||||
|
if len(started) > 0 {
|
||||||
|
return keepAll(started[0].ID)
|
||||||
|
}
|
||||||
if len(waiting) > 0 {
|
if len(waiting) > 0 {
|
||||||
// The waiting walk is the one open; the search walks after it, and the batches wait behind both.
|
// The waiting walk is the one open; the search walks after it, and the batches wait behind both.
|
||||||
return keepAll(waiting[0].ID)
|
return keepAll(waiting[0].ID)
|
||||||
}
|
}
|
||||||
return walkAlone(ctx, open, alone[0], now)
|
return walkAlone(ctx, open, alone[0], now)
|
||||||
}
|
}
|
||||||
if err := keepAll(""); err != nil {
|
return cutUnrelated(ctx, open, batches, started, waiting, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open
|
||||||
|
// ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4,
|
||||||
|
// issue 973). A batch cut in this look counts as open for the batches after it.
|
||||||
|
//
|
||||||
|
// - An own-path batch goes first and folds no waiting walk; a catalogue batch folds the waiting walks it is
|
||||||
|
// related to, and those it shares nothing with keep waiting.
|
||||||
|
// - **The controller's own path is related to everything** (decided for issue 973): a walk or batch that
|
||||||
|
// replaces the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside
|
||||||
|
// another, and nothing runs beside it.
|
||||||
|
// - A batch whose window is still open is assembling; one related to an open walk is queued behind it, and
|
||||||
|
// ADR 0276 decision 4 stands for it: nothing supersedes a walk that started.
|
||||||
|
func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting []inventory.Plan, now time.Time) error {
|
||||||
|
inv := open.inventory
|
||||||
|
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||||
|
var entries []inventory.Entry
|
||||||
|
var read map[string][]inventory.ReadRepository
|
||||||
|
var edges []inventory.Edge
|
||||||
|
judged := false
|
||||||
|
openWalks, err := openWalksOf(ctx, inv, started)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
|
||||||
// the catalogue batch queues behind it rather than being cut and overtaken). Several of a kind may be open
|
|
||||||
// since a merge that shares nothing is cut at once (ADR 0304 decision 1): walks stay one at a time, so one
|
|
||||||
// is cut per look, and once its walk started the others are queued behind it.
|
|
||||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
|
||||||
for i := range batches {
|
for i := range batches {
|
||||||
b := &batches[i]
|
b := &batches[i]
|
||||||
if !windowClosed(*b, now) {
|
if !windowClosed(*b, now) {
|
||||||
continue
|
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||||
}
|
|
||||||
if b.OwnPath() {
|
|
||||||
// A walk on the controller's own path folds no walk waiting for its word (decided during the build,
|
|
||||||
// 2026-10-10): that walk keeps waiting beside it, and starts once this one ended (advanceOnce).
|
|
||||||
waiting = nil
|
|
||||||
}
|
|
||||||
if err := cutBatch(ctx, open, b, waiting, now); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// The batches left wait behind the walk just cut when it started; the next look does the same.
|
|
||||||
if b.Open() && !b.Waiting() {
|
|
||||||
if batches, err = inv.Batches(ctx); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return keepAll(b.ID)
|
continue
|
||||||
|
}
|
||||||
|
if !judged {
|
||||||
|
var err error
|
||||||
|
if entries, err = inv.Catalogued(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if read, err = readForPlanning(ctx, inv); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if edges, err = inv.Dependencies(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
judged = true
|
||||||
|
}
|
||||||
|
held, err := inv.MergesOf(ctx, b.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fp := footprintAsCut(held, entries, read, edges)
|
||||||
|
var fold, keep []inventory.Plan
|
||||||
|
if !b.OwnPath() {
|
||||||
|
for _, w := range waiting {
|
||||||
|
ww, err := openWalkOf(ctx, inv, w)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if ok, _ := related(fp, ww.fp, edges); ok {
|
||||||
|
fold = append(fold, w)
|
||||||
|
fp = unionFootprint(fp, ww.fp)
|
||||||
|
} else {
|
||||||
|
keep = append(keep, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if behind, why := relatedOpen(fp, b.OwnPath(), openWalks, edges); behind != "" {
|
||||||
|
if err := keepBatchBecause(ctx, inv, b, now, behind, why); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := keepBatch(ctx, inv, b, now, ""); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := cutBatch(ctx, open, b, fold, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
waiting = keep
|
||||||
|
if c := windowClockOf(); c != nil {
|
||||||
|
c.Forget(ctx, b.ID) // a batch cut has no window left to close
|
||||||
|
}
|
||||||
|
cut, err := inv.PlanByID(ctx, b.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !cut.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w, err := openWalkOf(ctx, inv, cut)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cut.Waiting() {
|
||||||
|
waiting = append(waiting, cut)
|
||||||
|
} else {
|
||||||
|
openWalks = append(openWalks, w)
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1396,7 +1459,11 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
|||||||
grouped += " (on the controller's own path)"
|
grouped += " (on the controller's own path)"
|
||||||
}
|
}
|
||||||
if b.State == inventory.PlanQueued {
|
if b.State == inventory.PlanQueued {
|
||||||
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
because := ""
|
||||||
|
if w.BehindWhy != "" {
|
||||||
|
because = " (" + w.BehindWhy + ")"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("queued behind %s%s; grouped: %s; plan not yet calculated", w.Behind, because, grouped)
|
||||||
}
|
}
|
||||||
if w.CutAtOnce {
|
if w.CutAtOnce {
|
||||||
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
||||||
|
|||||||
@@ -334,8 +334,8 @@ func captured(t *testing.T, run func() error) string {
|
|||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// One walk at a time: a merge heard while a walk runs joins the next batch, which is cut when the walk ends;
|
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
||||||
// the walk that started is never superseded.
|
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
||||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
asksWithPaths(t)
|
asksWithPaths(t)
|
||||||
@@ -346,7 +346,7 @@ func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
|||||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||||
}
|
}
|
||||||
first := ws[0]
|
first := ws[0]
|
||||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||||
cutAt(t, open, t0.Add(5*time.Minute))
|
cutAt(t, open, t0.Add(5*time.Minute))
|
||||||
ws, bs := walks(t, open)
|
ws, bs := walks(t, open)
|
||||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||||
@@ -391,7 +391,7 @@ func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
|||||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||||
}
|
}
|
||||||
waiting := ws[0]
|
waiting := ws[0]
|
||||||
hear(t, open, repoMerge("two", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||||
ws, bs := walks(t, open)
|
ws, bs := walks(t, open)
|
||||||
if len(bs) != 0 || len(ws) != 2 {
|
if len(bs) != 0 || len(ws) != 2 {
|
||||||
@@ -405,7 +405,7 @@ func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
|||||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||||
}
|
}
|
||||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c1" || newer.CommitOf("novox/two") != "c2" {
|
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
||||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||||
}
|
}
|
||||||
if _, in := newer.Modules["one"]; !in {
|
if _, in := newer.Modules["one"]; !in {
|
||||||
@@ -662,8 +662,8 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// One walk at a time holds against the delivery's word too: a waiting walk let go beside a started one takes the
|
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
||||||
// word and starts once that one ended, asking nothing before.
|
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
||||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
asked := asksWithPaths(t)
|
asked := asksWithPaths(t)
|
||||||
@@ -671,8 +671,8 @@ func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
|||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
started, _ := walks(t, open)
|
started, _ := walks(t, open)
|
||||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/two", Branch: "main", Commit: "d1", Created: t0,
|
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"two"}}, Modules: map[string]*inventory.PlanModule{"two": {}},
|
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
||||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -729,49 +729,6 @@ func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The catch-up hands over what the bus lost after its branch's later merges were cut, and the record keeps it:
|
|
||||||
// an earlier merge is named by the walk that carried it; a merge made before a cut and heard after it, which
|
|
||||||
// no later merge carries, joins the next batch — neither reads as history and is dropped (review of this change).
|
|
||||||
func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
|
||||||
hear(t, open, repoMerge("one", "c2", base.Add(10*time.Second)), base.Add(11*time.Second))
|
|
||||||
cutAt(t, open, base.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
w := ws[0]
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
lost := []link.AnnouncedMerge{
|
|
||||||
{SourceMoved: repoMerge("one", "c1", base), At: base.Add(time.Second)},
|
|
||||||
{SourceMoved: repoMerge("one", "c3lost", base.Add(20*time.Second)), At: base.Add(21 * time.Second)},
|
|
||||||
}
|
|
||||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
read, err := readForPlanning(ctx, open.inventory)
|
|
||||||
return entries, read, err
|
|
||||||
}
|
|
||||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
|
||||||
(following{open: open}).SourceMoved, t.Logf); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
|
||||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
|
||||||
Commit: "c1", Carried: "c2"}) {
|
|
||||||
t.Fatalf("the earlier merge the catch-up handed over is not named by the walk that carried it: %+v",
|
|
||||||
got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
if _, kept, err := open.inventory.MergeOf(ctx, "novox/one", "c3lost"); err != nil || !kept {
|
|
||||||
t.Fatalf("the merge made before the cut and heard after it was dropped: %v %v", kept, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
||||||
|
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
||||||
|
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
cutAt(t, open, t0.Add(90*time.Second))
|
||||||
|
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
||||||
|
t.Fatalf("the first walk is not open: %+v", ws)
|
||||||
|
}
|
||||||
|
heard := t0.Add(100 * time.Second)
|
||||||
|
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 2 || len(bs) != 0 {
|
||||||
|
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
for _, w := range ws {
|
||||||
|
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
||||||
|
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
||||||
|
deferred(w), askedAny(w), w.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
||||||
|
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
||||||
|
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
cutAt(t, open, t0.Add(90*time.Second))
|
||||||
|
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
||||||
|
heard := t0.Add(100 * time.Second)
|
||||||
|
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||||
|
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 2 || len(bs) != 1 {
|
||||||
|
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
b := bs[0]
|
||||||
|
words := batchWords(b, heard)
|
||||||
|
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
||||||
|
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
||||||
|
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
||||||
|
}
|
||||||
|
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
||||||
|
t.Fatalf("the started walk was %s by the related merge", got.State)
|
||||||
|
}
|
||||||
|
first.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, heard.Add(2*time.Second))
|
||||||
|
if _, bs := walks(t, open); len(bs) != 0 {
|
||||||
|
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
||||||
|
// the two share (issue 973): it replaces the controller, so it goes alone.
|
||||||
|
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||||
|
cutAt(t, open, t0.Add(90*time.Second))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
first := ws[0]
|
||||||
|
heard := t0.Add(100 * time.Second)
|
||||||
|
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
||||||
|
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
||||||
|
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
||||||
|
}
|
||||||
|
first.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cutAt(t, open, heard.Add(time.Second))
|
||||||
|
ws, _ = walks(t, open)
|
||||||
|
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||||
|
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
||||||
|
}
|
||||||
|
own := ws[0]
|
||||||
|
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
||||||
|
later := heard.Add(time.Minute)
|
||||||
|
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
||||||
|
ws, bs = walks(t, open)
|
||||||
|
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
||||||
|
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -191,8 +191,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
if err := ask.Ask(ctx, request); err != nil {
|
if err := ask.Ask(ctx, request); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// A new head ends the check of the old one (novox/hq issue 726).
|
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||||
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
|
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||||
|
// of the forge here (issue 726).
|
||||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||||
if scope.gated() {
|
if scope.gated() {
|
||||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||||
@@ -270,7 +271,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
Held: heldBy(ctx),
|
Held: heldBy(ctx),
|
||||||
Seats: seatBases(ctx),
|
Seats: seatBases(ctx),
|
||||||
Source: sourceOnSeat(source),
|
Source: sourceOnSeat(source),
|
||||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
||||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||||
Release: release, Running: runningBy(ctx)},
|
Release: release, Running: runningBy(ctx)},
|
||||||
@@ -582,6 +583,13 @@ const maxCheckReport = 60 << 10
|
|||||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||||
func checked(ctx context.Context, result link.BuildResult) {
|
func checked(ctx context.Context, result link.BuildResult) {
|
||||||
|
// **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or
|
||||||
|
// the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red
|
||||||
|
// statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way.
|
||||||
|
if result.Failed == link.SupersededByKey {
|
||||||
|
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
||||||
|
return
|
||||||
|
}
|
||||||
sayChecked(ctx, checkedOf(result))
|
sayChecked(ctx, checkedOf(result))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one
|
||||||
|
// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the
|
||||||
|
// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again
|
||||||
|
// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing
|
||||||
|
// asks the forge, and the queue is read back whole.
|
||||||
|
func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
js, _ := aBuildQueue(t)
|
||||||
|
servingBus.Store(js)
|
||||||
|
t.Cleanup(func() { servingBus.Store(nil) })
|
||||||
|
holdTheDeliverySeat(t, open)
|
||||||
|
was := askDeliveryOwner
|
||||||
|
t.Cleanup(func() { askDeliveryOwner = was })
|
||||||
|
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
|
||||||
|
if verb != "groups" {
|
||||||
|
t.Fatalf("asked the owner %s", verb)
|
||||||
|
}
|
||||||
|
return json.RawMessage(`[]`), nil
|
||||||
|
}
|
||||||
|
// The repository the two pull requests are into, as a module the mesh builds from it.
|
||||||
|
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"}
|
||||||
|
check := func(number int, trunk string) {
|
||||||
|
t.Helper()
|
||||||
|
p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller",
|
||||||
|
Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true,
|
||||||
|
BaseSHA: trunk, Ready: number == 8}.pull()
|
||||||
|
p.MergeCheck, p.MergeCheckSaid = true, true
|
||||||
|
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Announced against the trunk as it stands, then asked again after each of two merges.
|
||||||
|
for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222",
|
||||||
|
"3333333333333333333333333333333333333333"} {
|
||||||
|
check(7, trunk)
|
||||||
|
check(8, trunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seen := map[int]string{}
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
c := a.Request.Check
|
||||||
|
if c == nil {
|
||||||
|
t.Fatalf("%s is no check", a.ID)
|
||||||
|
}
|
||||||
|
if _, twice := seen[c.Number]; twice {
|
||||||
|
t.Errorf("pull request %d has two queued checks", c.Number)
|
||||||
|
}
|
||||||
|
seen[c.Number] = c.Trunk
|
||||||
|
if a.State != link.AskWaiting {
|
||||||
|
t.Errorf("%s is %s", a.ID, a.State)
|
||||||
|
}
|
||||||
|
if want := "3333333333333333333333333333333333333333"; c.Trunk != want {
|
||||||
|
t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(seen) != 2 {
|
||||||
|
t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen)
|
||||||
|
}
|
||||||
|
// By the streams' own count: the ready one on its stream, the other on its.
|
||||||
|
for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} {
|
||||||
|
info, err := js.Context().StreamInfo(stream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.State.Msgs != want {
|
||||||
|
t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,9 +1,16 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -288,3 +295,47 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
|||||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
||||||
|
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
||||||
|
func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) {
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
was := checkEvents
|
||||||
|
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||||
|
t.Cleanup(func() { checkEvents = was })
|
||||||
|
heard := make(chan *nats.Msg, 4)
|
||||||
|
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3}
|
||||||
|
checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey})
|
||||||
|
checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||||
|
select {
|
||||||
|
case m := <-heard:
|
||||||
|
var c link.Checked
|
||||||
|
_ = json.Unmarshal(m.Data, &c)
|
||||||
|
if c.ID != "broken" || c.Verdict != "error" {
|
||||||
|
t.Fatalf("the first verdict said is %+v; want only the broken check's error", c)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("nothing was said")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case m := <-heard:
|
||||||
|
t.Fatalf("a second verdict was said: %s", m.Data)
|
||||||
|
case <-time.After(300 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -136,8 +136,9 @@ func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (i
|
|||||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||||
p.Delivery.Go.Local().Format("15:04:05"))
|
p.Delivery.Go.Local().Format("15:04:05"))
|
||||||
}
|
}
|
||||||
// **One walk at a time** (novox/hq ADR 0276): the word is taken, and the walk starts once no other walk is
|
// **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): the word is
|
||||||
// started (advanceOnce), so the delivery's owner says it once and is not refused.
|
// taken, and the walk starts once no started walk it is related to is open (advanceOnce), so the delivery's
|
||||||
|
// owner says it once and is not refused.
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||||
@@ -268,7 +269,7 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why
|
|||||||
append(moves, foldMoves...)); err != nil {
|
append(moves, foldMoves...)); err != nil {
|
||||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
||||||
}
|
}
|
||||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
// What the batch holds and names now: written again by the next look should this fail.
|
||||||
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
|
||||||
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
|
||||||
}
|
}
|
||||||
@@ -583,6 +584,9 @@ type orderMember struct {
|
|||||||
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
|
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
|
||||||
// a verdict no build judges names when the head is checked again through mesh-delivery.
|
// a verdict no build judges names when the head is checked again through mesh-delivery.
|
||||||
BaseSHA string `json:"base_sha,omitempty"`
|
BaseSHA string `json:"base_sha,omitempty"`
|
||||||
|
// Ready says the member's pull request is marked ready (novox/hq ADR 0406 §3, issue 974): its check is queued before
|
||||||
|
// every other check. Said by mesh-delivery on a ready mark.
|
||||||
|
Ready bool `json:"ready,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// pull is the member as the forge announced it.
|
// pull is the member as the forge announced it.
|
||||||
@@ -594,7 +598,7 @@ func (m orderMember) pull() link.PullUpdated {
|
|||||||
}
|
}
|
||||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
|
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
|
||||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready}
|
||||||
}
|
}
|
||||||
|
|
||||||
// orderPair is one "before" among a group's members, why, and the rules it won over.
|
// orderPair is one "before" among a group's members, why, and the rules it won over.
|
||||||
@@ -891,6 +895,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||||
group := set.String("group", "", "a delivery group's id")
|
group := set.String("group", "", "a delivery group's id")
|
||||||
|
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
||||||
|
"(novox/hq ADR 0406 §3)")
|
||||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||||
why := set.String("why", "", "why")
|
why := set.String("why", "", "why")
|
||||||
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
|
||||||
@@ -906,6 +912,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||||
return fmt.Errorf("the members are not readable: %w", err)
|
return fmt.Errorf("the members are not readable: %w", err)
|
||||||
}
|
}
|
||||||
|
// Marked ready by the verb's word for every member it names, besides any member that says so itself.
|
||||||
|
if *ready {
|
||||||
|
for i := range members {
|
||||||
|
members[i].Ready = true
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -976,7 +988,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
|
||||||
short(p.Commit), *by)
|
short(p.Commit), *by)
|
||||||
return nil
|
return nil
|
||||||
case "stop":
|
case "stop":
|
||||||
@@ -1165,6 +1177,8 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
request.Check.Group = group
|
request.Check.Group = group
|
||||||
|
// A group's check is queued as ready when any member's pull request is marked ready.
|
||||||
|
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
|
||||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
|||||||
@@ -287,6 +287,12 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
|||||||
return snapshot.Facts{}, err
|
return snapshot.Facts{}, err
|
||||||
}
|
}
|
||||||
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
||||||
|
// Whether its node-engine takes handed files is its contract (issue 969).
|
||||||
|
if stated, had, err := inv.HealthOf(ctx, n.Name); err != nil {
|
||||||
|
return snapshot.Facts{}, err
|
||||||
|
} else if had {
|
||||||
|
m.Contract = stated.Contract
|
||||||
|
}
|
||||||
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return snapshot.Facts{}, err
|
return snapshot.Facts{}, err
|
||||||
|
|||||||
@@ -227,8 +227,9 @@ func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// With a walk open, a batch cut at once still waits for it, said queued behind it (walks stay one at a time, ADR
|
// With a walk open that it is related to (the same repository, issue 973), a batch cut at once still waits for
|
||||||
// 0276 decision 4); it is cut once that walk ended, with no window to wait for.
|
// it, said queued behind it (ADR 0276 decision 4, ADR 0407 decision 4); it is cut once that walk ended, with no
|
||||||
|
// window to wait for.
|
||||||
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
||||||
open := windowed(t)
|
open := windowed(t)
|
||||||
groupsWaiting(t, "pr/c1")
|
groupsWaiting(t, "pr/c1")
|
||||||
@@ -241,7 +242,7 @@ func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
|||||||
}
|
}
|
||||||
first := ws[0]
|
first := ws[0]
|
||||||
heard := t0.Add(100 * time.Second)
|
heard := t0.Add(100 * time.Second)
|
||||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||||
ws, bs := walks(t, open)
|
ws, bs := walks(t, open)
|
||||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
|
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
|
||||||
!bs[0].Delivery.Batch.CutAtOnce {
|
!bs[0].Delivery.Batch.CutAtOnce {
|
||||||
@@ -280,7 +281,8 @@ func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testi
|
|||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
ws, _ := walks(t, open)
|
ws, _ := walks(t, open)
|
||||||
walk := ws[0]
|
walk := ws[0]
|
||||||
hear(t, open, repoMerge("two", "two1cccccccc", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
// Related to the walk by its repository (queued behind it), and to nothing the stop walks again.
|
||||||
|
hear(t, open, catalogueMerge("cat2cccccccc", "notes", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
||||||
_, bs := walks(t, open)
|
_, bs := walks(t, open)
|
||||||
if len(bs) != 1 {
|
if len(bs) != 1 {
|
||||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
||||||
|
|||||||
@@ -1017,6 +1017,13 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
|
|||||||
"kernel": m.Kernel, "capabilities": caps}); err != nil {
|
"kernel": m.Kernel, "capabilities": caps}); err != nil {
|
||||||
return notes, err
|
return notes, err
|
||||||
}
|
}
|
||||||
|
// What its node-engine takes, as the mesh composes it today (issue 969): without the statement the gate
|
||||||
|
// composes a moved module the old way on a machine the mesh hands it to.
|
||||||
|
if stated, ok := standInHealth(m, f.Taken); ok {
|
||||||
|
if _, err := inv.RecordHealth(ctx, stated); err != nil {
|
||||||
|
return notes, err
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, record := range []func(context.Context, string, string) error{inv.RecordSealingKey, inv.RecordOverlayKey} {
|
for _, record := range []func(context.Context, string, string) error{inv.RecordSealingKey, inv.RecordOverlayKey} {
|
||||||
key, err := standInKey()
|
key, err := standInKey()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1355,3 +1362,12 @@ func firstOr(items, otherwise []string) string {
|
|||||||
}
|
}
|
||||||
return "it says nothing more"
|
return "it says nothing more"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// standInHealth is the statement the gate gives a machine of the snapshot so that it composes as the mesh does:
|
||||||
|
// only its contract, which says whether it takes handed files (issue 969). False for a snapshot that carries none.
|
||||||
|
func standInHealth(m snapshot.Machine, taken time.Time) (inventory.NodeHealth, bool) {
|
||||||
|
if m.Contract == 0 {
|
||||||
|
return inventory.NodeHealth{}, false
|
||||||
|
}
|
||||||
|
return inventory.NodeHealth{Node: m.Name, Contract: m.Contract, SaidAt: taken, HeardAt: taken}, true
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 969: the gate composes a machine as the mesh does, so a moved module is handed where the machine takes
|
||||||
|
// handed files and composed the old way where it does not.
|
||||||
|
func TestIssue969TheGateGivesAMachineItsContractSoItTakesHandedFiles(t *testing.T) {
|
||||||
|
taken := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
stated, ok := standInHealth(snapshot.Machine{Name: "f35", Contract: link.HandedContract}, taken)
|
||||||
|
if !ok || !takesHanded(stated, ok) {
|
||||||
|
t.Fatalf("a machine whose engine takes handed files must be given that in the gate: %+v, %v", stated, ok)
|
||||||
|
}
|
||||||
|
old, ok := standInHealth(snapshot.Machine{Name: "e12", Contract: link.HandedContract - 1}, taken)
|
||||||
|
if !ok || takesHanded(old, ok) {
|
||||||
|
t.Fatalf("a machine whose engine is older must not take handed files: %+v, %v", old, ok)
|
||||||
|
}
|
||||||
|
none, ok := standInHealth(snapshot.Machine{Name: "d01"}, taken)
|
||||||
|
if ok || takesHanded(none, ok) {
|
||||||
|
t.Fatalf("a snapshot that carries no contract states nothing: %+v, %v", none, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 969, through the store: a machine whose engine stated the handed contract is in the snapshot with it, the
|
||||||
|
// gate raises it as taking handed files, and one whose statement is below it is raised as not taking them. Run in
|
||||||
|
// the merge check with the other tests that need a store; the composition itself, handed here and refused there, is
|
||||||
|
// the catalogue's TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded.
|
||||||
|
func TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
said := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
|
||||||
|
for node, contract := range map[string]int{"anchor": link.HandedContract, "laptop": link.HandedContract - 1} {
|
||||||
|
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: contract,
|
||||||
|
SaidAt: said, HeardAt: said}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(f.Machines) != 2 {
|
||||||
|
t.Fatalf("the snapshot holds %d machines", len(f.Machines))
|
||||||
|
}
|
||||||
|
want := map[string]bool{}
|
||||||
|
for _, m := range f.Machines {
|
||||||
|
if m.Contract == 0 {
|
||||||
|
t.Fatalf("%s: the snapshot carries no contract", m.Name)
|
||||||
|
}
|
||||||
|
want[m.Name] = m.Contract >= link.HandedContract
|
||||||
|
}
|
||||||
|
handing := 0
|
||||||
|
for _, hands := range want {
|
||||||
|
if hands {
|
||||||
|
handing++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if handing != 1 {
|
||||||
|
t.Fatalf("one machine takes handed files in the snapshot, got %v", want)
|
||||||
|
}
|
||||||
|
|
||||||
|
raised, drop, err := throwawayStores(ctx, os.Getenv("MESH_TEST_POSTGRES"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer drop()
|
||||||
|
if _, err := raiseFromFacts(ctx, raised, f, map[string]catalogue.Manifest{}, map[string]inventory.Source{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, hands := range want {
|
||||||
|
got, err := engineTakesHanded(ctx, raised.inventory, name)
|
||||||
|
if err != nil || got != hands {
|
||||||
|
t.Errorf("%s: the gate raises it as taking handed files = %v (%v), the mesh says %v", name, got, err, hands)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The merge path whole, against a real bus and the store (issue 973 and issue 921, novox/hq ADR 0407 decisions 1
|
||||||
|
// and 3; ADR 0391's proof): the forge's merged event is published on the subject the controller consumes, the
|
||||||
|
// controller's server hands it over and the window keys are on the bus. Store-backed: skips without one.
|
||||||
|
|
||||||
|
// noListener hears no node: these tests send merges only.
|
||||||
|
type noListener struct{ link.Listener }
|
||||||
|
|
||||||
|
// theMergePathOn serves the controller's consume loop and its merge window on a test bus, and returns a way to
|
||||||
|
// publish the forge's merged event.
|
||||||
|
func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) {
|
||||||
|
t.Helper()
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
server := link.ConnectNats(js, nil, noListener{})
|
||||||
|
if err := server.Follows(following{open: open}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = server.Serve(ctx) }()
|
||||||
|
if err := startMergeWindow(ctx, open, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-done
|
||||||
|
setWindowClock(nil)
|
||||||
|
windowLook.Store(nil)
|
||||||
|
})
|
||||||
|
return func(m link.SourceMoved) {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.Context().Publish(broker.ControllerFollows[3], body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func becomes(t *testing.T, within time.Duration, what string, is func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(within)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if is() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("%s did not happen within %s", what, within)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeKept(open *stores, repo, commit string) bool {
|
||||||
|
_, known, err := open.inventory.MergeOf(context.Background(), repo, commit)
|
||||||
|
return err == nil && known
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two merges of one repository heard while another controller holds the plans, and a rollout of that repository
|
||||||
|
// open, are both recorded once the plans are free (handed over again by the bus, never dropped), join one batch,
|
||||||
|
// and the next rollout carries both: no person released anything (issue 921).
|
||||||
|
func TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
was, wasPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||||
|
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 20*time.Millisecond
|
||||||
|
t.Cleanup(func() { inventory.HoldWaitFor, inventory.HoldPoll = was, wasPoll })
|
||||||
|
// A window of a second: a merge joining a batch opens its window again (ADR 0407 decision 1), so the batch the
|
||||||
|
// two held merges make is cut when that window has closed and the rollout before it has ended, not before.
|
||||||
|
settings := t.TempDir() + "/merge-window.json"
|
||||||
|
if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(mergeWindowFileVar, settings)
|
||||||
|
publish := theMergePathOn(t, open)
|
||||||
|
ctx := t.Context()
|
||||||
|
|
||||||
|
publish(repoMerge("one", "c1", time.Now()))
|
||||||
|
becomes(t, 15*time.Second, "the first rollout opening", func() bool {
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
return len(ws) == 1 && ws[0].Open()
|
||||||
|
})
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
first := ws[0]
|
||||||
|
|
||||||
|
release, err := open.inventory.HoldPlans(ctx, true) // the other controller works the plans
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
publish(repoMerge("one", "c2", time.Now()))
|
||||||
|
publish(repoMerge("one", "c3", time.Now()))
|
||||||
|
time.Sleep(2500 * time.Millisecond) // both met the hold and were given back to the bus
|
||||||
|
if mergeKept(open, "novox/one", "c2") || mergeKept(open, "novox/one", "c3") {
|
||||||
|
t.Fatal("a merge was recorded while another holder had the plans")
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
becomes(t, 30*time.Second, "both merges recorded after the hold", func() bool {
|
||||||
|
return mergeKept(open, "novox/one", "c2") && mergeKept(open, "novox/one", "c3")
|
||||||
|
})
|
||||||
|
|
||||||
|
first.State = inventory.PlanDone
|
||||||
|
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The rollout ended: the look that cuts what waited behind it, once the batch's window is closed too.
|
||||||
|
var bs []inventory.Plan
|
||||||
|
becomes(t, 15*time.Second, "the next rollout being cut with both merges", func() bool {
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
ws, bs = walks(t, open)
|
||||||
|
return len(bs) == 0 && len(ws) == 2
|
||||||
|
})
|
||||||
|
if ws[0].CommitOf("novox/one") != "c3" || len(ws[0].Delivery.Merges) != 2 {
|
||||||
|
t.Fatalf("the next rollout does not carry both merges: commit %q, merges %+v", ws[0].CommitOf("novox/one"),
|
||||||
|
ws[0].Delivery.Merges)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A batch's window closes by the expiry of its key on the bus, heard by the watcher, and the batch is cut with
|
||||||
|
// no cutter and no call to cut (ADR 0407 decision 1): a window of a second, a merge that waits for it.
|
||||||
|
func TestABatchWindowClosesByTheExpiryOfItsKey(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
everyHeadWaits(t)
|
||||||
|
settings := t.TempDir() + "/merge-window.json"
|
||||||
|
if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(mergeWindowFileVar, settings)
|
||||||
|
publish := theMergePathOn(t, open)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
publish(repoMerge("one", "c1", start))
|
||||||
|
becomes(t, 10*time.Second, "the merge waiting in a batch", func() bool {
|
||||||
|
_, bs := walks(t, open)
|
||||||
|
return len(bs) == 1 && bs[0].State == inventory.PlanAssembling
|
||||||
|
})
|
||||||
|
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||||
|
t.Fatalf("the batch was cut before its window closed: %+v", ws)
|
||||||
|
}
|
||||||
|
becomes(t, 15*time.Second, "the batch being cut by its key's expiry", func() bool {
|
||||||
|
ws, bs := walks(t, open)
|
||||||
|
return len(ws) == 1 && len(bs) == 0
|
||||||
|
})
|
||||||
|
if took := time.Since(start); took < time.Second {
|
||||||
|
t.Fatalf("the batch was cut after %s, before its window of a second was up", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The merge window's clock is on the bus (novox/hq ADR 0407 decision 1, issue 973): an open batch has a key
|
||||||
|
// `<batch>.quiet` that lives the window and is made again by each merge into the batch, and a key `<batch>.most`
|
||||||
|
// made once when the batch opens that lives the window's most. The first of the two to expire closes the batch,
|
||||||
|
// and the watcher hears it: no timer polls the batches. The store still keeps the times (ClosesAt, AtMost) as the
|
||||||
|
// record of when a window closed, which `plans` reads; the keys are what wakes the controller.
|
||||||
|
|
||||||
|
// windowClock is the bus's side of the merge window.
|
||||||
|
type windowClock interface {
|
||||||
|
// Arm makes a batch's keys: its quiet key again, to live quiet; its most key if it has none, to live most.
|
||||||
|
Arm(ctx context.Context, batch string, quiet, most time.Duration) error
|
||||||
|
// Forget takes a batch's keys away: it was cut, or closed, and nothing is to wake the controller for it.
|
||||||
|
Forget(ctx context.Context, batch string)
|
||||||
|
}
|
||||||
|
|
||||||
|
// windowLook is the look a window's expiry takes: the watcher's, and the fallback timer's.
|
||||||
|
var windowLook atomic.Pointer[func(batch string)]
|
||||||
|
|
||||||
|
var (
|
||||||
|
windowMu sync.RWMutex
|
||||||
|
windowKeyed windowClock // nil in a process that is not the serving controller
|
||||||
|
)
|
||||||
|
|
||||||
|
// setWindowClock makes clock the merge window's; nil takes it away.
|
||||||
|
func setWindowClock(c windowClock) {
|
||||||
|
windowMu.Lock()
|
||||||
|
defer windowMu.Unlock()
|
||||||
|
windowKeyed = c
|
||||||
|
}
|
||||||
|
|
||||||
|
func windowClockOf() windowClock {
|
||||||
|
windowMu.RLock()
|
||||||
|
defer windowMu.RUnlock()
|
||||||
|
return windowKeyed
|
||||||
|
}
|
||||||
|
|
||||||
|
// windowDeadlines are how long a batch's two keys are to live from now: what is left of its window and of its
|
||||||
|
// most, each at least a second (the least a key can live). Not armed for a batch whose window is closed already
|
||||||
|
// (by a stop, by its time, or because it is cut at once) or that is no batch.
|
||||||
|
func windowDeadlines(b inventory.Plan, now time.Time) (quiet, most time.Duration, armed bool) {
|
||||||
|
if b.Delivery == nil || b.Delivery.Batch == nil || windowClosed(b, now) {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
// Rounded up to a whole second, at least one: the server rounds a key's time to live down to whole seconds, so
|
||||||
|
// a fraction would close the key before the record's time and the look would cut nothing.
|
||||||
|
least := func(d time.Duration) time.Duration {
|
||||||
|
if d < time.Second {
|
||||||
|
return time.Second
|
||||||
|
}
|
||||||
|
return (d + time.Second - 1) / time.Second * time.Second
|
||||||
|
}
|
||||||
|
w := b.Delivery.Batch
|
||||||
|
return least(w.ClosesAt.Sub(now)), least(w.AtMost.Sub(now)), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// armWindow puts a batch's clock on the bus after its record was kept. Where the bus will not take it the window
|
||||||
|
// closes by a timer of this batch alone, once, as long as is left: never silently never.
|
||||||
|
func armWindow(ctx context.Context, b inventory.Plan, now time.Time) {
|
||||||
|
c := windowClockOf()
|
||||||
|
quiet, most, armed := windowDeadlines(b, now)
|
||||||
|
if c == nil || !armed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := c.Arm(ctx, b.ID, quiet, most); err != nil {
|
||||||
|
fmt.Printf("batches: the window of %s could not be put on the bus (%v); it closes by a timer of its own\n", b.ID, err)
|
||||||
|
if look := windowLook.Load(); look != nil {
|
||||||
|
time.AfterFunc(quiet, func() { (*look)(b.ID) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// busWindow is the clock on the controller's bucket.
|
||||||
|
type busWindow struct {
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBusWindow opens the bucket and starts the watcher, which names each batch whose key expired to look until
|
||||||
|
// the context ends. Only expiry wakes it: a key taken away by Forget or made again by Arm leaves a delete.
|
||||||
|
func newBusWindow(ctx context.Context, api jetstream.JetStream, look func(batch string)) (*busWindow, error) {
|
||||||
|
kv, err := api.KeyValue(ctx, broker.MergeWindowBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the merge window's bucket: %w", err)
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("watching the merge window: %w", err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer w.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case e, ok := <-w.Updates():
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if e == nil || e.Operation() != jetstream.KeyValuePurge {
|
||||||
|
continue // a put, or a delete this controller made
|
||||||
|
}
|
||||||
|
if batch, _, found := strings.Cut(e.Key(), "."); found && look != nil {
|
||||||
|
look(batch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return &busWindow{kv: kv}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *busWindow) Arm(ctx context.Context, batch string, quiet, most time.Duration) error {
|
||||||
|
// The most is made once, when the batch opens: it is there already for every merge after the first.
|
||||||
|
if _, err := w.kv.Create(ctx, batch+".most", []byte("most"), jetstream.KeyTTL(most)); err != nil &&
|
||||||
|
!errors.Is(err, jetstream.ErrKeyExists) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The quiet key is made again, not changed: a key's time to live is set when it is made.
|
||||||
|
if err := w.kv.Delete(ctx, batch+".quiet"); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := w.kv.Create(ctx, batch+".quiet", []byte("quiet"), jetstream.KeyTTL(quiet)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *busWindow) Forget(ctx context.Context, batch string) {
|
||||||
|
for _, key := range []string{batch + ".quiet", batch + ".most"} {
|
||||||
|
_ = w.kv.Delete(ctx, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
|
||||||
|
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
|
||||||
|
func lookAtBatches(ctx context.Context, open *stores) {
|
||||||
|
release, err := open.inventory.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||||
|
fmt.Printf("batches: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startMergeWindow puts the merge window's clock on the bus for as long as the context lives: the controller is
|
||||||
|
// the lease holder here, the only one that writes it. The watcher looks at the batches when a key expires; the
|
||||||
|
// batches open now get their keys, and one look is taken, so a window that closed while no controller served is
|
||||||
|
// cut (ADR 0407 decision 1). No timer polls them.
|
||||||
|
func startMergeWindow(ctx context.Context, open *stores, js *broker.JetStream) error {
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
look := func(string) { lookAtBatches(ctx, open) }
|
||||||
|
windowLook.Store(&look)
|
||||||
|
clock, err := newBusWindow(ctx, api, look)
|
||||||
|
if err != nil {
|
||||||
|
windowLook.Store(nil)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
setWindowClock(clock)
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
setWindowClock(nil)
|
||||||
|
windowLook.Store(nil)
|
||||||
|
}()
|
||||||
|
release, err := open.inventory.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
batches, err := open.inventory.Batches(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
for _, b := range batches {
|
||||||
|
armWindow(ctx, b, now)
|
||||||
|
}
|
||||||
|
return cutBatchesHeld(ctx, open, now)
|
||||||
|
}
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// windowOnABus is the merge window's clock on a real bus, and the batches its watcher heard expire.
|
||||||
|
func windowOnABus(t *testing.T) (*busWindow, <-chan string) {
|
||||||
|
t.Helper()
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(conn.Close)
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
heard := make(chan string, 16)
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
t.Cleanup(stop)
|
||||||
|
w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return w, heard
|
||||||
|
}
|
||||||
|
|
||||||
|
func hears(t *testing.T, heard <-chan string, within time.Duration) (string, bool) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case b := <-heard:
|
||||||
|
return b, true
|
||||||
|
case <-time.After(within):
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A batch's quiet key expires when no merge came for the window, and the watcher names the batch (issue 973,
|
||||||
|
// novox/hq ADR 0407 decision 1).
|
||||||
|
func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) {
|
||||||
|
w, heard := windowOnABus(t)
|
||||||
|
if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" {
|
||||||
|
t.Fatalf("heard %q (%v); want plan-1", b, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge into the batch makes the quiet key again, so the window restarts: it does not expire at the first
|
||||||
|
// deadline, and the batch's most closes it all the same.
|
||||||
|
func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) {
|
||||||
|
w, heard := windowOnABus(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(1200 * time.Millisecond)
|
||||||
|
if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if b, ok := hears(t, heard, 1500*time.Millisecond); ok {
|
||||||
|
t.Fatalf("heard %q before the restarted window was up", b)
|
||||||
|
}
|
||||||
|
// Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s.
|
||||||
|
start := time.Now()
|
||||||
|
for time.Since(start) < 2500*time.Millisecond {
|
||||||
|
if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(700 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" {
|
||||||
|
t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A batch cut is forgotten: its keys are taken away by a delete, which is no expiry and wakes nobody.
|
||||||
|
func TestForgettingABatchWakesNobody(t *testing.T) {
|
||||||
|
w, heard := windowOnABus(t)
|
||||||
|
if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Forget(t.Context(), "plan-3")
|
||||||
|
if b, ok := hears(t, heard, 3*time.Second); ok {
|
||||||
|
t.Fatalf("a forgotten batch was heard expire: %q", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The deadlines a batch's keys are made with: what is left of its window and of its most, at least a second;
|
||||||
|
// none for a batch whose window is closed or that is no batch.
|
||||||
|
func TestTheDeadlinesOfABatchsKeys(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||||
|
batch := func(closes, most time.Duration, closed bool) inventory.Plan {
|
||||||
|
return inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||||
|
ClosesAt: now.Add(closes), AtMost: now.Add(most), Closed: closed}}}
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
b inventory.Plan
|
||||||
|
quiet, most time.Duration
|
||||||
|
armed bool
|
||||||
|
}{
|
||||||
|
{"open", batch(90*time.Second, 10*time.Minute, false), 90 * time.Second, 10 * time.Minute, true},
|
||||||
|
{"almost up", batch(100*time.Millisecond, 5*time.Second, false), time.Second, 5 * time.Second, true},
|
||||||
|
{"closed by a stop", batch(90*time.Second, 10*time.Minute, true), 0, 0, false},
|
||||||
|
{"up", batch(-time.Second, 10*time.Minute, false), 0, 0, false},
|
||||||
|
{"cut at once", func() inventory.Plan {
|
||||||
|
b := batch(90*time.Second, 10*time.Minute, false)
|
||||||
|
b.Delivery.Batch.CutAtOnce = true
|
||||||
|
return b
|
||||||
|
}(), 0, 0, false},
|
||||||
|
{"no batch", inventory.Plan{}, 0, 0, false},
|
||||||
|
} {
|
||||||
|
quiet, most, armed := windowDeadlines(c.b, now)
|
||||||
|
if armed != c.armed || quiet != c.quiet || most != c.most {
|
||||||
|
t.Errorf("%s: %v, %v, %v; want %v, %v, %v", c.what, quiet, most, armed, c.quiet, c.most, c.armed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A deadline that is not a whole number of seconds is rounded up, never down: the server rounds a key's time to live
|
||||||
|
// down to whole seconds, so 89.999999 s would expire at 89 s, before the window is closed by the record's times, and
|
||||||
|
// the look would cut nothing (review of issue 973).
|
||||||
|
func TestAFractionalDeadlineIsRoundedUpToTheNextWholeSecond(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||||
|
b := inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{
|
||||||
|
ClosesAt: now.Add(89*time.Second + 999999*time.Microsecond), AtMost: now.Add(599*time.Second + time.Nanosecond)}}}
|
||||||
|
quiet, most, armed := windowDeadlines(b, now)
|
||||||
|
if !armed || quiet != 90*time.Second || most != 600*time.Second {
|
||||||
|
t.Errorf("deadlines %v, %v (armed %v); want 1m30s, 10m0s", quiet, most, armed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,243 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
|
||||||
//
|
|
||||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
|
||||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
|
||||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
|
||||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
|
||||||
// built from that repository stayed behind and were built by hand.
|
|
||||||
//
|
|
||||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
|
||||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
|
||||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
|
||||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
|
||||||
// would still move something was never acted on — it is said, and acted on now.
|
|
||||||
const (
|
|
||||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
|
||||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
|
||||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
|
||||||
mergeGrace = 10 * time.Minute
|
|
||||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
|
||||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
|
||||||
mergeLookBack = 24 * time.Hour
|
|
||||||
// mergeCatchUpEvery is how often the stream is read back.
|
|
||||||
mergeCatchUpEvery = 5 * time.Minute
|
|
||||||
)
|
|
||||||
|
|
||||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
|
||||||
type merges interface {
|
|
||||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// unheardMerges is the announcements of merges the controller has not heard (novox/hq ADR 0276): a merge kept
|
|
||||||
// in a batch is not acted on until its batch is cut, which can be past mergeGrace behind a long walk, and is
|
|
||||||
// not missed for that.
|
|
||||||
type unheardMerges struct {
|
|
||||||
merges
|
|
||||||
inv *inventory.Inventory
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u unheardMerges) AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
|
||||||
all, err := u.merges.AnnouncedMerges(ctx, since)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []link.AnnouncedMerge
|
|
||||||
for _, a := range all {
|
|
||||||
_, kept, err := u.inv.MergeOf(ctx, a.Owner+"/"+a.Repo, a.Commit)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !kept {
|
|
||||||
out = append(out, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// owedToTheRecord says a merge read as history is still owed to the record (novox/hq ADR 0276): the merges
|
|
||||||
// reader knows, when it keeps them (unheardMerges).
|
|
||||||
func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved, entries []inventory.Entry,
|
|
||||||
read map[string][]inventory.ReadRepository) bool {
|
|
||||||
u, ok := announced.(unheardMerges)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
raw := m
|
|
||||||
raw.MergedAt = ""
|
|
||||||
owed, err := owedLate(ctx, u.inv, m, merged, wouldMove(raw, entries, read))
|
|
||||||
return err == nil && owed
|
|
||||||
}
|
|
||||||
|
|
||||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
|
||||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
|
||||||
f := following{open: open}
|
|
||||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
read, err := readForPlanning(ctx, open.inventory)
|
|
||||||
return entries, read, err
|
|
||||||
}
|
|
||||||
failing := ""
|
|
||||||
tick := time.NewTicker(mergeCatchUpEvery)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
}
|
|
||||||
watchedMerges.begin()
|
|
||||||
err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announced, open.inventory}, catalogued, f.SourceMoved, func(format string, args ...any) {
|
|
||||||
fmt.Printf(format+"\n", args...)
|
|
||||||
})
|
|
||||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
|
||||||
// says that instead, and leaves what the last one found standing.
|
|
||||||
watchedMerges.end(time.Now(), err)
|
|
||||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
|
||||||
why := ""
|
|
||||||
if err != nil {
|
|
||||||
why = err.Error()
|
|
||||||
}
|
|
||||||
if why != failing {
|
|
||||||
if why != "" {
|
|
||||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
|
||||||
} else {
|
|
||||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
|
||||||
}
|
|
||||||
failing = why
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
|
||||||
// move something is said and acted on, oldest first.
|
|
||||||
//
|
|
||||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
|
||||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
|
||||||
// would have.
|
|
||||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|
||||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
|
||||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
|
||||||
|
|
||||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
entries, read, err := catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, a := range all {
|
|
||||||
// A merge the forge said no time of is dated by its announcement, so a packaging module's look can
|
|
||||||
// make it history once acted on, and the catch-up does not act on it again every pass (ADR 0267).
|
|
||||||
if a.SourceMoved.MergedAt == "" && !a.At.IsZero() {
|
|
||||||
a.SourceMoved.MergedAt = a.At.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
if now.Sub(a.At) < mergeGrace {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 && !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if entries, read, err = catalogued(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
moves := wouldMove(a.SourceMoved, entries, read)
|
|
||||||
if len(moves) == 0 {
|
|
||||||
if !owedToTheRecord(ctx, announced, a.SourceMoved, entries, read) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
raw := a.SourceMoved
|
|
||||||
raw.MergedAt = ""
|
|
||||||
moves = wouldMove(raw, entries, read)
|
|
||||||
}
|
|
||||||
var names []string
|
|
||||||
for _, e := range moves {
|
|
||||||
names = append(names, e.Manifest.Module)
|
|
||||||
}
|
|
||||||
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
|
||||||
At: a.At, Modules: names})
|
|
||||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
|
||||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
|
||||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
|
||||||
isAre(len(names)))
|
|
||||||
if err := act(ctx, a.SourceMoved); err != nil {
|
|
||||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
|
||||||
a.Owner, a.Repo, a.Commit, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if entries, read, err = catalogued(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
|
||||||
type missedMerge struct {
|
|
||||||
Owner, Repo, Base, Commit string
|
|
||||||
// At is when the bus took the announcement.
|
|
||||||
At time.Time
|
|
||||||
Modules []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
|
||||||
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
|
||||||
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
|
||||||
// the fault. The next pass, finding it acted on, clears it.
|
|
||||||
type mergeWatch struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
passed time.Time
|
|
||||||
err error
|
|
||||||
finding []missedMerge
|
|
||||||
missed []missedMerge
|
|
||||||
}
|
|
||||||
|
|
||||||
var watchedMerges = &mergeWatch{}
|
|
||||||
|
|
||||||
func (w *mergeWatch) begin() {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
w.finding = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *mergeWatch) found(m missedMerge) {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
w.finding = append(w.finding, m)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *mergeWatch) end(at time.Time, err error) {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
w.passed, w.err = at, err
|
|
||||||
if err == nil {
|
|
||||||
w.missed = w.finding
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
|
||||||
// could not read.
|
|
||||||
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
|
||||||
}
|
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// announcedList is the events stream's merges as a test gives them.
|
|
||||||
type announcedList []link.AnnouncedMerge
|
|
||||||
|
|
||||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
|
||||||
var out []link.AnnouncedMerge
|
|
||||||
for _, m := range a {
|
|
||||||
if !m.At.Before(since) {
|
|
||||||
out = append(out, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
|
||||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
|
||||||
type aCatalogue struct {
|
|
||||||
entries []inventory.Entry
|
|
||||||
acted []string
|
|
||||||
fail error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
|
||||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
|
||||||
return func(_ context.Context, m link.SourceMoved) error {
|
|
||||||
if c.fail != nil {
|
|
||||||
return c.fail
|
|
||||||
}
|
|
||||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
|
||||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
|
||||||
for i := range c.entries {
|
|
||||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
|
||||||
c.entries[i].Source.Head = m.Commit
|
|
||||||
c.entries[i].Source.Seen = now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func at(s string) time.Time {
|
|
||||||
t, err := time.Parse(time.RFC3339, s)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
|
||||||
return link.AnnouncedMerge{
|
|
||||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
|
||||||
MergedAt: mergedAt, Paths: paths,
|
|
||||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
|
||||||
At: at(onTheBus),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
|
||||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
|
||||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
|
|
||||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
|
||||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
|
||||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
|
||||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
|
||||||
// has had its time with it.
|
|
||||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
|
||||||
cat := &aCatalogue{entries: []inventory.Entry{
|
|
||||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
|
||||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
|
||||||
// Acted on when it was announced: looked at after it was merged.
|
|
||||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
|
||||||
}}
|
|
||||||
stream := announcedList{
|
|
||||||
// Nothing the mesh holds is built from the records repository.
|
|
||||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
|
||||||
// Acted on: its module was looked at since.
|
|
||||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
|
||||||
// Never handed over.
|
|
||||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
|
||||||
"node-tools/internal/console/console.go"),
|
|
||||||
}
|
|
||||||
var said []string
|
|
||||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
|
||||||
clock := at("2026-10-05T22:50:00Z")
|
|
||||||
now := func() time.Time { return clock }
|
|
||||||
pass := func() {
|
|
||||||
t.Helper()
|
|
||||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pass()
|
|
||||||
if len(cat.acted) != 0 {
|
|
||||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
|
||||||
}
|
|
||||||
|
|
||||||
clock = at("2026-10-05T22:58:00Z")
|
|
||||||
pass()
|
|
||||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
|
||||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
|
||||||
}
|
|
||||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
|
||||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
|
||||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
|
||||||
}
|
|
||||||
|
|
||||||
clock = at("2026-10-05T23:03:00Z")
|
|
||||||
pass()
|
|
||||||
if len(cat.acted) != 1 || len(said) != 1 {
|
|
||||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
|
||||||
// that could not be acted on is said and tried again on the next pass.
|
|
||||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
|
||||||
cat := &aCatalogue{
|
|
||||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
|
||||||
fail: errors.New("the store is restarting"),
|
|
||||||
}
|
|
||||||
stream := announcedList{
|
|
||||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
|
||||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
|
||||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
|
||||||
}
|
|
||||||
var said []string
|
|
||||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
|
||||||
clock := at("2026-10-05T22:00:00Z")
|
|
||||||
now := func() time.Time { return clock }
|
|
||||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
|
||||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
|
||||||
}
|
|
||||||
cat.fail = nil
|
|
||||||
clock = at("2026-10-05T22:05:00Z")
|
|
||||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
|
||||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
|
||||||
// missed it, and acting on it days later would be a surprise rebuild.
|
|
||||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
|
||||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
|
||||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
|
||||||
clock := at("2026-10-05T22:00:00Z")
|
|
||||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
|
||||||
func(string, ...any) {}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(cat.acted) != 0 {
|
|
||||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -923,8 +923,10 @@ func brokerReachableAt(ctx context.Context, inv *inventory.Inventory, known brok
|
|||||||
// module's own secret named broker; a module that declares none has nothing to read it with, and
|
// module's own secret named broker; a module that declares none has nothing to read it with, and
|
||||||
// an account nothing reads is an orphan on the bus (novox/hq 04-ISSUES/078). Said before the
|
// an account nothing reads is an orphan on the bus (novox/hq 04-ISSUES/078). Said before the
|
||||||
// account is made, not after.
|
// account is made, not after.
|
||||||
|
// A module that runs its machine's provisioner client and declares no broker is issued the client's account
|
||||||
|
// alone (issue 917): the client speaks on the bus only as itself.
|
||||||
func mayIssue(m catalogue.Manifest) error {
|
func mayIssue(m catalogue.Manifest) error {
|
||||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) {
|
||||||
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
||||||
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
||||||
}
|
}
|
||||||
@@ -943,14 +945,6 @@ func mayIssue(m catalogue.Manifest) error {
|
|||||||
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||||
node, busAddress string) error {
|
node, busAddress string) error {
|
||||||
|
|
||||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
|
||||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
|
||||||
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Where the module is told to find the bus, and what certificate it must present. The same pair
|
// Where the module is told to find the bus, and what certificate it must present. The same pair
|
||||||
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
|
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
|
||||||
// trust store, so an address alone fails at TLS.
|
// trust store, so an address alone fails at TLS.
|
||||||
@@ -962,8 +956,17 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
|
if _, reads := m.OwnSecrets["broker"]; reads {
|
||||||
return err
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !runsTheProvisionerClient(m) {
|
if !runsTheProvisionerClient(m) {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -429,12 +429,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
"out) has not let it start yet.",
|
"out) has not let it start yet.",
|
||||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||||
}),
|
}),
|
||||||
"merge-not-acted": worded(func(o conditions.Observation) words {
|
|
||||||
return words{Headline: "A merge was never picked up",
|
|
||||||
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
|
||||||
"sent. The mesh catches up on missed merges by itself.",
|
|
||||||
Resolved: "Resolved: the merge was picked up"}
|
|
||||||
}),
|
|
||||||
"ask-lost": worded(func(o conditions.Observation) words {
|
"ask-lost": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "A build has no result",
|
return words{Headline: "A build has no result",
|
||||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||||
|
|||||||
@@ -1126,7 +1126,13 @@ func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node strin
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
return had && stated.Contract >= link.HandedContract, nil
|
return takesHanded(stated, had), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// takesHanded is whether a machine's newest statement (had: it ever stated one) is of an engine that writes a
|
||||||
|
// filled file on the provisioner client's word alone.
|
||||||
|
func takesHanded(stated inventory.NodeHealth, had bool) bool {
|
||||||
|
return had && stated.Contract >= link.HandedContract
|
||||||
}
|
}
|
||||||
|
|
||||||
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
|
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
|
||||||
|
|||||||
@@ -0,0 +1,674 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on
|
||||||
|
// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a
|
||||||
|
// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for
|
||||||
|
// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are
|
||||||
|
// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is
|
||||||
|
// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be
|
||||||
|
// (armWalkEnd).
|
||||||
|
|
||||||
|
// looker runs the look at the plans that events ask for.
|
||||||
|
type looker struct {
|
||||||
|
signal chan struct{} // one place: asks made together are one look
|
||||||
|
look func(ctx context.Context) (failed bool)
|
||||||
|
// gap is the least time between two looks, so a plan that kept changing could not make the worker spin.
|
||||||
|
gap time.Duration
|
||||||
|
// first runs before the first look: what a controller that took the lease sets up.
|
||||||
|
first func(ctx context.Context)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
retry *time.Timer
|
||||||
|
backoff time.Duration
|
||||||
|
base time.Duration
|
||||||
|
most time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
retryLookFirst = 5 * time.Second
|
||||||
|
retryLookMost = time.Minute
|
||||||
|
lookGap = 200 * time.Millisecond
|
||||||
|
)
|
||||||
|
|
||||||
|
func newLooker(look func(ctx context.Context) bool) *looker {
|
||||||
|
return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ask has a look run, soon; never blocks.
|
||||||
|
func (l *looker) ask() {
|
||||||
|
select {
|
||||||
|
case l.signal <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// run takes the asks one at a time until the context ends.
|
||||||
|
func (l *looker) run(ctx context.Context) {
|
||||||
|
if l.first != nil {
|
||||||
|
l.first(ctx)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
l.mu.Lock()
|
||||||
|
if l.retry != nil {
|
||||||
|
l.retry.Stop()
|
||||||
|
}
|
||||||
|
l.mu.Unlock()
|
||||||
|
return
|
||||||
|
case <-l.signal:
|
||||||
|
}
|
||||||
|
l.after(l.look(ctx))
|
||||||
|
if l.gap > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-time.After(l.gap):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures.
|
||||||
|
func (l *looker) after(failed bool) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
if !failed {
|
||||||
|
l.backoff = 0
|
||||||
|
if l.retry != nil {
|
||||||
|
l.retry.Stop()
|
||||||
|
l.retry = nil
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if l.retry != nil {
|
||||||
|
return // one is armed already
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case l.backoff == 0:
|
||||||
|
l.backoff = l.base
|
||||||
|
case l.backoff*2 > l.most:
|
||||||
|
l.backoff = l.most
|
||||||
|
default:
|
||||||
|
l.backoff *= 2
|
||||||
|
}
|
||||||
|
// retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by
|
||||||
|
// the first look that fails nothing; the step is otherwise started by an event, and a failure is none.
|
||||||
|
l.retry = time.AfterFunc(l.backoff, func() {
|
||||||
|
l.mu.Lock()
|
||||||
|
l.retry = nil
|
||||||
|
l.mu.Unlock()
|
||||||
|
l.ask()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// theLooker is the serving controller's; nil in a command, whose looks are its own.
|
||||||
|
var (
|
||||||
|
lookerMu sync.RWMutex
|
||||||
|
theLooker *looker
|
||||||
|
)
|
||||||
|
|
||||||
|
// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send,
|
||||||
|
// a silent key expired. Nothing in a process that is not serving.
|
||||||
|
func askToLook() {
|
||||||
|
lookerMu.RLock()
|
||||||
|
l := theLooker
|
||||||
|
lookerMu.RUnlock()
|
||||||
|
if l != nil {
|
||||||
|
l.ask()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
|
||||||
|
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
|
||||||
|
func startPlanLooks(ctx context.Context, open *stores) {
|
||||||
|
l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
|
||||||
|
l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
|
||||||
|
lookerMu.Lock()
|
||||||
|
theLooker = l
|
||||||
|
lookerMu.Unlock()
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
lookerMu.Lock()
|
||||||
|
if theLooker == l {
|
||||||
|
theLooker = nil
|
||||||
|
}
|
||||||
|
lookerMu.Unlock()
|
||||||
|
}()
|
||||||
|
go l.run(ctx)
|
||||||
|
l.ask()
|
||||||
|
}
|
||||||
|
|
||||||
|
// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's
|
||||||
|
// ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour.
|
||||||
|
var applySilentAfter = inventory.ApplySilentAfter
|
||||||
|
|
||||||
|
// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974):
|
||||||
|
// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own
|
||||||
|
// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left
|
||||||
|
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one.
|
||||||
|
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) {
|
||||||
|
if err := startRolloutSilent(ctx, js); err != nil {
|
||||||
|
fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
|
||||||
|
}
|
||||||
|
if err := startProvAppliedLooks(ctx, js); err != nil {
|
||||||
|
fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err)
|
||||||
|
}
|
||||||
|
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||||
|
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
||||||
|
}
|
||||||
|
startPlanLooks(ctx, open)
|
||||||
|
}
|
||||||
|
|
||||||
|
// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a
|
||||||
|
// send kept or a command's move is followed by the look that asks the next tier, in place of the timer that did), and,
|
||||||
|
// for a walk that ended, its merges' check builds forgotten (novox/hq ADR 0350 §5).
|
||||||
|
func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inventory.Inventory, inventory.Plan) {
|
||||||
|
return func(_ *inventory.Inventory, p inventory.Plan) {
|
||||||
|
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||||
|
planKept(p)
|
||||||
|
if p.WalkSaid == inventory.WalkEnded {
|
||||||
|
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open.
|
||||||
|
const planLookPasses = 6
|
||||||
|
|
||||||
|
// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is
|
||||||
|
// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the
|
||||||
|
// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is
|
||||||
|
// answered once.
|
||||||
|
func lookAtPlans(ctx context.Context, open *stores) (failed bool) {
|
||||||
|
release, err := open.inventory.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("plans: cannot hold them: %v\n", err)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
func() {
|
||||||
|
defer release()
|
||||||
|
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||||
|
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||||
|
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
// Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a
|
||||||
|
// batch cut makes one.
|
||||||
|
for pass := 0; pass < planLookPasses; pass++ {
|
||||||
|
before := openPlansNow(ctx, open.inventory)
|
||||||
|
if advanceHeld(ctx, open) {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
if openPlansNow(ctx, open.inventory) == before {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := syncSilentKeys(ctx, open); err != nil {
|
||||||
|
fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err)
|
||||||
|
failed = true // so the look is retried
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
keepWalkPhases(ctx, open)
|
||||||
|
return failed
|
||||||
|
}
|
||||||
|
|
||||||
|
// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a
|
||||||
|
// single step rather than the look (the tests of the steps).
|
||||||
|
func advancePlans(ctx context.Context, open *stores) {
|
||||||
|
release, err := open.inventory.HoldPlans(ctx, false)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, inventory.ErrPlansBusy) {
|
||||||
|
fmt.Printf("plans: cannot hold them: %v\n", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||||
|
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||||
|
}
|
||||||
|
advanceHeld(ctx, open)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not.
|
||||||
|
func openPlansNow(ctx context.Context, inv *inventory.Inventory) string {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "unreadable " + time.Now().String()
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(plans))
|
||||||
|
for _, p := range plans {
|
||||||
|
ids = append(ids, p.ID+":"+string(p.State))
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
return strings.Join(ids, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the wait for a machine's report, as keys on the bus ---
|
||||||
|
|
||||||
|
var (
|
||||||
|
silentMu sync.RWMutex
|
||||||
|
silentKeyed silentClock // nil in a process that is not the serving controller
|
||||||
|
)
|
||||||
|
|
||||||
|
// setSilentClock makes clock the rollout's; nil takes it away.
|
||||||
|
func setSilentClock(c silentClock) {
|
||||||
|
silentMu.Lock()
|
||||||
|
defer silentMu.Unlock()
|
||||||
|
silentKeyed = c
|
||||||
|
}
|
||||||
|
|
||||||
|
func silentClockOf() silentClock {
|
||||||
|
silentMu.RLock()
|
||||||
|
defer silentMu.RUnlock()
|
||||||
|
return silentKeyed
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent
|
||||||
|
// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine
|
||||||
|
// judges, and the machines a release's gates wait on. Pure.
|
||||||
|
func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time {
|
||||||
|
waits := map[silentWait]time.Time{}
|
||||||
|
put := func(w silentWait, at time.Time) {
|
||||||
|
if at.IsZero() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if was, ok := waits[w]; !ok || at.Before(was) {
|
||||||
|
waits[w] = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !p.Open() {
|
||||||
|
return waits
|
||||||
|
}
|
||||||
|
if r := p.Release; r != nil {
|
||||||
|
gates := []*inventory.PlanGate{r.Gate}
|
||||||
|
for _, g := range r.Gates {
|
||||||
|
gates = append(gates, g)
|
||||||
|
}
|
||||||
|
for _, g := range gates {
|
||||||
|
if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range g.Machines {
|
||||||
|
put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return waits
|
||||||
|
}
|
||||||
|
if p.Waiting() {
|
||||||
|
return waits
|
||||||
|
}
|
||||||
|
tierOf := func(module string) int {
|
||||||
|
for i, tier := range p.Tiers {
|
||||||
|
if slices.Contains(tier, module) {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
waiting, _, _ := walkReports(p, reports, now, prov)
|
||||||
|
for module, s := range p.Modules {
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range waiting {
|
||||||
|
sent, ok := s.Rest[node]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at := sent.At
|
||||||
|
if at.IsZero() && s.SentAt != nil {
|
||||||
|
at = *s.SentAt
|
||||||
|
}
|
||||||
|
put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at)
|
||||||
|
}
|
||||||
|
if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil {
|
||||||
|
for _, node := range g.Machines {
|
||||||
|
put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return waits
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the
|
||||||
|
// look it wakes finds the machine silent and not a fraction early.
|
||||||
|
func silentTTL(sent, now time.Time) time.Duration {
|
||||||
|
left := sent.Add(applySilentAfter).Sub(now) + time.Second
|
||||||
|
if left < time.Second {
|
||||||
|
return time.Second
|
||||||
|
}
|
||||||
|
return left
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the
|
||||||
|
// key is of an earlier send) and the keys to delete (the wait is over). Pure.
|
||||||
|
func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) {
|
||||||
|
for w, sent := range want {
|
||||||
|
if was, ok := held[w]; !ok || !was.Equal(sent) {
|
||||||
|
arm = append(arm, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for w := range held {
|
||||||
|
if _, ok := want[w]; !ok {
|
||||||
|
drop = append(drop, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return arm, drop
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out
|
||||||
|
// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the
|
||||||
|
// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the
|
||||||
|
// look alone knows the report was on that send). Only the lease holder looks, so only it writes.
|
||||||
|
func syncSilentKeys(ctx context.Context, open *stores) error {
|
||||||
|
c := silentClockOf()
|
||||||
|
inv := open.inventory
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
want := map[silentWait]time.Time{}
|
||||||
|
for _, p := range plans {
|
||||||
|
for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) {
|
||||||
|
want[w] = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keepSilentKeys(ctx, c, want, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
||||||
|
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
|
||||||
|
// the look is retried.
|
||||||
|
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
||||||
|
byTimer := func() {
|
||||||
|
for w, sent := range want {
|
||||||
|
silentByTimer(w, sent, silentTTL(sent, now))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c == nil {
|
||||||
|
byTimer()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
held, err := c.Held(ctx)
|
||||||
|
if err != nil {
|
||||||
|
byTimer()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
arm, drop := silentDiff(want, held)
|
||||||
|
var first error
|
||||||
|
for _, w := range arm {
|
||||||
|
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
||||||
|
silentByTimer(w, want[w], silentTTL(want[w], now))
|
||||||
|
if first == nil {
|
||||||
|
first = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, w := range drop {
|
||||||
|
if err := c.Reported(ctx, w); err != nil && first == nil {
|
||||||
|
first = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
var silentTimers = struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
timers map[silentWait]time.Time
|
||||||
|
}{timers: map[silentWait]time.Time{}}
|
||||||
|
|
||||||
|
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
|
||||||
|
// (DegradableGrants says why that can be): never silently never. Once for each send.
|
||||||
|
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
|
||||||
|
silentTimers.mu.Lock()
|
||||||
|
defer silentTimers.mu.Unlock()
|
||||||
|
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
silentTimers.timers[w] = sent
|
||||||
|
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
|
||||||
|
time.AfterFunc(ttl, func() {
|
||||||
|
silentTimers.mu.Lock()
|
||||||
|
if silentTimers.timers[w].Equal(sent) {
|
||||||
|
delete(silentTimers.timers, w)
|
||||||
|
}
|
||||||
|
silentTimers.mu.Unlock()
|
||||||
|
askToLook()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
||||||
|
// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires.
|
||||||
|
func startRolloutSilent(ctx context.Context, js *broker.JetStream) error {
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() })
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
setSilentClock(clock)
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
silentMu.Lock()
|
||||||
|
if silentKeyed == silentClock(clock) {
|
||||||
|
silentKeyed = nil
|
||||||
|
}
|
||||||
|
silentMu.Unlock()
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// startProvAppliedLooks has the plans looked at when a machine's provisioner client says a word (`prov.applied.<node>.<module>`):
|
||||||
|
// a send that handed modules to the provisioner is applied only once the client said so too, and without this the walk
|
||||||
|
// would wait for its silent key (ADR 0362, ADR 0406 §5, issue 974). New words only: the first look reads the rest.
|
||||||
|
func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stream, err := api.Stream(ctx, broker.ProvAppliedStream)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the provisioner clients' words: %w", err)
|
||||||
|
}
|
||||||
|
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("watching the provisioner clients' words: %w", err)
|
||||||
|
}
|
||||||
|
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("watching the provisioner clients' words: %w", err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
consuming.Stop()
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// startPlanMovedLooks has the plans looked at when any process says one moved: the serving controller follows its own
|
||||||
|
// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn,
|
||||||
|
// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the
|
||||||
|
// first look reads the rest. A look its own save asks is the same look: asks are one.
|
||||||
|
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error {
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stream, err := api.Stream(ctx, broker.EventsStream)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the events stream: %w", err)
|
||||||
|
}
|
||||||
|
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{
|
||||||
|
FilterSubjects: []string{link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved)},
|
||||||
|
DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("following the plans moved: %w", err)
|
||||||
|
}
|
||||||
|
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("following the plans moved: %w", err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
consuming.Stop()
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the end of a walk whose end is not known ---
|
||||||
|
|
||||||
|
var walkEnds = struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
timers map[string]*time.Timer
|
||||||
|
}{timers: map[string]*time.Timer{}}
|
||||||
|
|
||||||
|
// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded
|
||||||
|
// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at
|
||||||
|
// each save of the done walk, so it counts from the last.
|
||||||
|
func armWalkEnd(id string, savedAt time.Time) {
|
||||||
|
d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second))
|
||||||
|
if d <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
walkEnds.mu.Lock()
|
||||||
|
defer walkEnds.mu.Unlock()
|
||||||
|
if t := walkEnds.timers[id]; t != nil {
|
||||||
|
t.Stop()
|
||||||
|
}
|
||||||
|
// retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first
|
||||||
|
// look after the lease was taken arms it again for the walks still unrecorded.
|
||||||
|
walkEnds.timers[id] = time.AfterFunc(d, func() {
|
||||||
|
walkEnds.mu.Lock()
|
||||||
|
delete(walkEnds.timers, id)
|
||||||
|
walkEnds.mu.Unlock()
|
||||||
|
askToLook()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease.
|
||||||
|
func rearmWalkEnds(ctx context.Context, open *stores) {
|
||||||
|
recent, err := open.inventory.RecentPlans(ctx, 30)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, p := range recent {
|
||||||
|
if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour {
|
||||||
|
armWalkEnd(p.ID, p.Updated)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or
|
||||||
|
// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded.
|
||||||
|
func planKept(p inventory.Plan) {
|
||||||
|
if p.State == inventory.PlanDone && p.Release == nil {
|
||||||
|
armWalkEnd(p.ID, time.Now())
|
||||||
|
}
|
||||||
|
askToLook()
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- `sent`: a tier sent to a machine, said ---
|
||||||
|
|
||||||
|
// sendMade is one machine a plan's tier was sent to.
|
||||||
|
type sendMade struct {
|
||||||
|
Tier int
|
||||||
|
Node string
|
||||||
|
Modules []string
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendsMade collects the sends one step of a plan made, to be said once the plan is saved.
|
||||||
|
type sendsMade struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
list []sendMade
|
||||||
|
}
|
||||||
|
|
||||||
|
type sendsMadeKey struct{}
|
||||||
|
|
||||||
|
func withSendsMade(ctx context.Context, s *sendsMade) context.Context {
|
||||||
|
return context.WithValue(ctx, sendsMadeKey{}, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// noteSends records that modules of the plan's tier were sent to the nodes.
|
||||||
|
func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) {
|
||||||
|
s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade)
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
for _, n := range nodes {
|
||||||
|
s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine
|
||||||
|
// and the modules it was sent, and when.
|
||||||
|
type planSentBody struct {
|
||||||
|
Event string `json:"event"`
|
||||||
|
Plan string `json:"plan"`
|
||||||
|
Tier int `json:"tier"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Modules []string `json:"modules"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentBodies are the events a step's sends are said as.
|
||||||
|
func sentBodies(plan string, sends *sendsMade) []planSentBody {
|
||||||
|
sends.mu.Lock()
|
||||||
|
defer sends.mu.Unlock()
|
||||||
|
out := make([]planSentBody, 0, len(sends.list))
|
||||||
|
for _, s := range sends.list {
|
||||||
|
out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node,
|
||||||
|
Modules: s.Modules, At: s.At})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the
|
||||||
|
// controller's own next step does not wait for it.
|
||||||
|
func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) {
|
||||||
|
bus := checkEvents
|
||||||
|
if bus == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, body := range sentBodies(p.ID, sends) {
|
||||||
|
raw, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
|
defer stop()
|
||||||
|
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil {
|
||||||
|
fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,351 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) {
|
||||||
|
sent := time.Now().Add(-sentAgo).UTC()
|
||||||
|
p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0,
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
|
||||||
|
"b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
|
||||||
|
}}
|
||||||
|
r := inventory.Reported{Node: "ace", Sent: &sent}
|
||||||
|
if reported {
|
||||||
|
at := sent.Add(time.Minute)
|
||||||
|
r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied
|
||||||
|
}
|
||||||
|
return p, []inventory.Reported{r}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR
|
||||||
|
// 0406 §5); one that reported on the send, or that is silent already, is none.
|
||||||
|
func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
p, reports := rolloutOf(5*time.Minute, false)
|
||||||
|
waits := silentWaitsOf(p, reports, now, nil)
|
||||||
|
if len(waits) != 1 {
|
||||||
|
t.Fatalf("waits = %v; want the one machine", waits)
|
||||||
|
}
|
||||||
|
for w, at := range waits {
|
||||||
|
if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute {
|
||||||
|
t.Errorf("wait %+v sent at %v is wrong", w, at)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p, reports = rolloutOf(5*time.Minute, true)
|
||||||
|
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
|
||||||
|
t.Errorf("a machine that reported on its send is waited on: %v", waits)
|
||||||
|
}
|
||||||
|
p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false)
|
||||||
|
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
|
||||||
|
t.Errorf("a machine silent past its time is waited on again: %v", waits)
|
||||||
|
}
|
||||||
|
p, reports = rolloutOf(time.Minute, false)
|
||||||
|
p.State = inventory.PlanDone
|
||||||
|
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
|
||||||
|
t.Errorf("a plan that ended is waiting: %v", waits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait.
|
||||||
|
func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) {
|
||||||
|
since := time.Now().Add(-time.Minute).UTC()
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since}
|
||||||
|
p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}}
|
||||||
|
waits := silentWaitsOf(p, nil, time.Now(), nil)
|
||||||
|
if len(waits) != 2 {
|
||||||
|
t.Fatalf("waits = %v; want g14 and novox", waits)
|
||||||
|
}
|
||||||
|
g.Verdict = inventory.GatePassed
|
||||||
|
if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 {
|
||||||
|
t.Errorf("a gate with its verdict still waits: %v", waits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted.
|
||||||
|
func TestTheKeysFollowTheWaits(t *testing.T) {
|
||||||
|
t1 := time.Now().UTC()
|
||||||
|
t2 := t1.Add(time.Minute)
|
||||||
|
a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"}
|
||||||
|
arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1})
|
||||||
|
if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) {
|
||||||
|
t.Errorf("armed %v; want b (sent again) and c (no key)", arm)
|
||||||
|
}
|
||||||
|
if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) {
|
||||||
|
t.Errorf("dropped %v; want the key with no wait", drop)
|
||||||
|
}
|
||||||
|
if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second {
|
||||||
|
t.Errorf("a wait long over has a key of %v; want the least, one second", ttl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker {
|
||||||
|
l := newLooker(func(ctx context.Context) bool {
|
||||||
|
looks.Add(1)
|
||||||
|
if block != nil {
|
||||||
|
<-block
|
||||||
|
}
|
||||||
|
return failures.Add(-1) >= 0
|
||||||
|
})
|
||||||
|
l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
func eventually(t *testing.T, what string, ok func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
|
||||||
|
if ok() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("never: %s", what)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asks made while a look runs are not lost and are one look after it, not one each.
|
||||||
|
func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) {
|
||||||
|
var failures, looks atomic.Int32
|
||||||
|
block := make(chan struct{})
|
||||||
|
l := lookerCounting(&failures, &looks, block)
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
l.ask()
|
||||||
|
eventually(t, "the first look starts", func() bool { return looks.Load() == 1 })
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
l.ask()
|
||||||
|
}
|
||||||
|
block <- struct{}{} // the first look ends
|
||||||
|
eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 })
|
||||||
|
block <- struct{}{}
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
if n := looks.Load(); n != 2 {
|
||||||
|
t.Errorf("%d looks for 21 asks; want 2", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails
|
||||||
|
// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry).
|
||||||
|
func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) {
|
||||||
|
var failures, looks atomic.Int32
|
||||||
|
failures.Store(3) // three looks fail, then one is good
|
||||||
|
l := lookerCounting(&failures, &looks, nil)
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
l.ask()
|
||||||
|
eventually(t, "four looks", func() bool { return looks.Load() == 4 })
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
if n := looks.Load(); n != 4 {
|
||||||
|
t.Errorf("%d looks; the good one should have ended the retries at 4", n)
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
armed, backoff := l.retry != nil, l.backoff
|
||||||
|
l.mu.Unlock()
|
||||||
|
if armed || backoff != 0 {
|
||||||
|
t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRetryWaitDoublesToItsMost(t *testing.T) {
|
||||||
|
l := newLooker(func(context.Context) bool { return true })
|
||||||
|
l.base, l.most = 5*time.Second, time.Minute
|
||||||
|
var seen []time.Duration
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
l.after(true)
|
||||||
|
seen = append(seen, l.backoff)
|
||||||
|
l.retry.Stop()
|
||||||
|
l.retry = nil
|
||||||
|
}
|
||||||
|
want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute}
|
||||||
|
for i := range want {
|
||||||
|
if seen[i] != want[i] {
|
||||||
|
t.Fatalf("waits %v; want %v", seen, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known.
|
||||||
|
func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) {
|
||||||
|
var failures, looks atomic.Int32
|
||||||
|
l := lookerCounting(&failures, &looks, nil)
|
||||||
|
lookerMu.Lock()
|
||||||
|
theLooker = l
|
||||||
|
lookerMu.Unlock()
|
||||||
|
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
|
||||||
|
planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone})
|
||||||
|
t.Cleanup(func() {
|
||||||
|
walkEnds.mu.Lock()
|
||||||
|
defer walkEnds.mu.Unlock()
|
||||||
|
if t := walkEnds.timers["walk-1"]; t != nil {
|
||||||
|
t.Stop()
|
||||||
|
}
|
||||||
|
delete(walkEnds.timers, "walk-1")
|
||||||
|
})
|
||||||
|
eventually(t, "a look", func() bool { return looks.Load() == 1 })
|
||||||
|
walkEnds.mu.Lock()
|
||||||
|
_, armed := walkEnds.timers["walk-1"]
|
||||||
|
walkEnds.mu.Unlock()
|
||||||
|
if !armed {
|
||||||
|
t.Error("a walk saved done is not armed to be recorded")
|
||||||
|
}
|
||||||
|
askToLook() // no panic, no looker, once it is taken away
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tier-sends a step made are said once each, per machine, with the modules it was sent.
|
||||||
|
func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) {
|
||||||
|
sends := &sendsMade{}
|
||||||
|
ctx := withSendsMade(context.Background(), sends)
|
||||||
|
at := time.Now().UTC()
|
||||||
|
noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at)
|
||||||
|
noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at)
|
||||||
|
noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept
|
||||||
|
got := sentBodies("plan-9", sends)
|
||||||
|
if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" ||
|
||||||
|
got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" {
|
||||||
|
t.Errorf("sent bodies = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5).
|
||||||
|
func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) {
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var failures, looks atomic.Int32
|
||||||
|
l := lookerCounting(&failures, &looks, nil)
|
||||||
|
lookerMu.Lock()
|
||||||
|
theLooker = l
|
||||||
|
lookerMu.Unlock()
|
||||||
|
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) })
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
if err := startRolloutSilent(ctx, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c := silentClockOf()
|
||||||
|
w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"}
|
||||||
|
if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := c.Reported(ctx, w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(1800 * time.Millisecond)
|
||||||
|
if n := looks.Load(); n != 0 {
|
||||||
|
t.Fatalf("a key a report deleted asked for %d looks", n)
|
||||||
|
}
|
||||||
|
if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 })
|
||||||
|
}
|
||||||
|
|
||||||
|
// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only
|
||||||
|
// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said.
|
||||||
|
func TestAProvisionerClientsWordAsksForALook(t *testing.T) {
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var failures, looks atomic.Int32
|
||||||
|
l := lookerCounting(&failures, &looks, nil)
|
||||||
|
lookerMu.Lock()
|
||||||
|
theLooker = l
|
||||||
|
lookerMu.Unlock()
|
||||||
|
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
if err := startProvAppliedLooks(ctx, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 })
|
||||||
|
}
|
||||||
|
|
||||||
|
type brokenClock struct{ silentClock }
|
||||||
|
|
||||||
|
func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||||
|
return nil, errors.New("the bus is down")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
|
||||||
|
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
|
||||||
|
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
|
||||||
|
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||||
|
sent := time.Now().UTC()
|
||||||
|
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
||||||
|
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("no error given back")
|
||||||
|
}
|
||||||
|
silentTimers.mu.Lock()
|
||||||
|
armed := silentTimers.timers[w].Equal(sent)
|
||||||
|
silentTimers.mu.Unlock()
|
||||||
|
if !armed {
|
||||||
|
t.Error("no timer of its own watches the wait")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
||||||
|
// looked at: the controller follows its own plan-moved (issue 974, round 1).
|
||||||
|
func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) {
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var failures, looks atomic.Int32
|
||||||
|
l := lookerCounting(&failures, &looks, nil)
|
||||||
|
lookerMu.Lock()
|
||||||
|
theLooker = l
|
||||||
|
lookerMu.Unlock()
|
||||||
|
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 })
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||||
|
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
|
||||||
|
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
|
||||||
|
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
|
||||||
|
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||||
|
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||||
|
var found []string
|
||||||
|
for _, root := range []string{"../../cmd", "../../internal"} {
|
||||||
|
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
text := string(raw)
|
||||||
|
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||||
|
found = append(found, path+" writes the plan table")
|
||||||
|
}
|
||||||
|
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||||
|
found = append(found, path+" calls writePlan")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(found) > 0 {
|
||||||
|
t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n"))
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile("../../internal/inventory/plans.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
text := string(raw)
|
||||||
|
if n := len(write.FindAllString(text, -1)); n != 1 {
|
||||||
|
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
|
||||||
|
}
|
||||||
|
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
|
||||||
|
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||||
|
next := strings.Index(text[save:], "\nfunc ")
|
||||||
|
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
|
||||||
|
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
|
||||||
|
}
|
||||||
|
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
|
||||||
|
t.Error("SavePlansMoving does not say each plan it wrote")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// movesSaid records what the hook hears: for each plan, the revisions it was said at.
|
||||||
|
type movesSaid struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
revs map[string][]int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.revs[p.ID] = append(s.revs[p.ID], p.Revision)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved.
|
||||||
|
func sayingEverySave(t *testing.T) *movesSaid {
|
||||||
|
t.Helper()
|
||||||
|
s := &movesSaid{revs: map[string][]int64{}}
|
||||||
|
was := inventory.PlanSaved
|
||||||
|
inventory.PlanSaved = s.hear
|
||||||
|
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said.
|
||||||
|
func (s *movesSaid) noUnsaid(t *testing.T, open *stores) {
|
||||||
|
t.Helper()
|
||||||
|
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if len(recent) == 0 {
|
||||||
|
t.Fatal("no plan was kept")
|
||||||
|
}
|
||||||
|
for _, p := range recent {
|
||||||
|
revs := s.revs[p.ID]
|
||||||
|
if len(revs) == 0 || revs[len(revs)-1] != p.Revision {
|
||||||
|
t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that
|
||||||
|
// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2).
|
||||||
|
func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
everyHeadWaits(t)
|
||||||
|
asksWithPaths(t)
|
||||||
|
heard := sayingEverySave(t)
|
||||||
|
hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second))
|
||||||
|
if _, bs := walks(t, open); len(bs) != 1 {
|
||||||
|
t.Fatalf("%d batch(es) opened", len(bs))
|
||||||
|
}
|
||||||
|
heard.noUnsaid(t, open)
|
||||||
|
cutAt(t, open, t0.Add(91*time.Second))
|
||||||
|
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||||
|
t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs))
|
||||||
|
}
|
||||||
|
heard.noUnsaid(t, open)
|
||||||
|
counted := 0
|
||||||
|
for _, r := range heard.revs {
|
||||||
|
counted += len(r)
|
||||||
|
}
|
||||||
|
cutAt(t, open, t0.Add(92*time.Second))
|
||||||
|
again := 0
|
||||||
|
for _, r := range heard.revs {
|
||||||
|
again += len(r)
|
||||||
|
}
|
||||||
|
if again != counted {
|
||||||
|
t.Errorf("a look that changed nothing said %d more move(s)", again-counted)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -197,12 +197,23 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
|||||||
return oneWalkAtATime(p, plans)
|
return oneWalkAtATime(p, plans)
|
||||||
}
|
}
|
||||||
|
|
||||||
// oneWalkAtATime refuses a retry while another walk is open, started or waiting for its word (novox/hq ADR
|
// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq
|
||||||
// 0276): a walk retried beside it would be two walks at once.
|
// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same
|
||||||
|
// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it
|
||||||
|
// plans, which already hold their dependents, and the repositories it carries.
|
||||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
||||||
|
mine := footprintOfWalk(p, nil)
|
||||||
for _, q := range plans {
|
for _, q := range plans {
|
||||||
if q.ID != p.ID && q.Open() && q.Release == nil {
|
if q.ID == p.ID || !q.Open() || q.Release != nil {
|
||||||
return fmt.Errorf("%s is open (%s): one walk at a time — retry %s once it ended", q.ID, q.Named(), p.ID)
|
continue
|
||||||
|
}
|
||||||
|
if ownWalk(p) || ownWalk(q) {
|
||||||
|
return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+
|
||||||
|
"retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||||
|
}
|
||||||
|
if ok, why := related(mine, footprintOfWalk(q, nil), nil); ok {
|
||||||
|
return fmt.Errorf("%s is open (%s) and %s: one walk at a time beside a related one — retry %s once it ended",
|
||||||
|
q.ID, q.Named(), why, p.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -320,7 +320,11 @@ func probeAgentRoot(ctx context.Context, d *doctor) ([]conditions.Observation, e
|
|||||||
quiet, err := searchStillRunning(ctx, inv, node, n.AgentAccount, h, had, now)
|
quiet, err := searchStillRunning(ctx, inv, node, n.AgentAccount, h, had, now)
|
||||||
return err == nil && quiet
|
return err == nil && quiet
|
||||||
}
|
}
|
||||||
return rootObservations(ctx, r, trustedMachines(r.entries), time.Now()), nil
|
obs := rootObservations(ctx, r, trustedMachines(r.entries), time.Now())
|
||||||
|
// A membership that lost its verified sender to a judgement that failed at a push gets it back here, once
|
||||||
|
// the machines pass (issue 930); the bound is one self-check.
|
||||||
|
giveVerifiedSendersBack(ctx, d)
|
||||||
|
return obs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rootObservations judges the machines and says each that fails.
|
// rootObservations judges the machines and says each that fails.
|
||||||
@@ -365,14 +369,3 @@ func rootFreeAnswer(ctx context.Context, machines string, now time.Time) (any, e
|
|||||||
}
|
}
|
||||||
return map[string]any{"machines": judgeRootFree(ctx, newRootReader(ctx, d.open.inventory, conn), names, now)}, nil
|
return map[string]any{"machines": judgeRootFree(ctx, newRootReader(ctx, d.open.inventory, conn), names, now)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rootFreeNow is the machines judged root-free, for composing a push's memberships: only those that pass.
|
|
||||||
func rootFreeNow(ctx context.Context, r *rootReader, machines []string, now time.Time) map[string]bool {
|
|
||||||
free := map[string]bool{}
|
|
||||||
for _, v := range judgeRootFree(ctx, r, machines, now) {
|
|
||||||
if v.Free {
|
|
||||||
free[v.Machine] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return free
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
@@ -60,3 +61,32 @@ func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *te
|
|||||||
t.Fatal("a module with nowhere to read the client's account is read as running it")
|
t.Fatal("a module with nowhere to read the client's account is read as running it")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A module that runs the client and declares no broker of its own, as the catalogue's mesh-provisioner-client does,
|
||||||
|
// may be issued: the client's account alone (issue 917). Before, `module issue` refused it for having no broker,
|
||||||
|
// while a push refused it for having no account, so no machine could run the client.
|
||||||
|
func TestAClientWithoutABrokerOfItsOwnMayBeIssued(t *testing.T) {
|
||||||
|
if err := mayIssue(aClientModule()); err != nil {
|
||||||
|
t.Fatalf("the client alone is refused an account: %v", err)
|
||||||
|
}
|
||||||
|
neither := aBot("bot", false)
|
||||||
|
if err := mayIssue(neither); err == nil {
|
||||||
|
t.Fatal("a module with nowhere to read any account may be issued one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assigning the client tries to issue the client's account, as assigning any module issues its broker (issue 917):
|
||||||
|
// without a bus to issue on, the line says so and names the verb, never nothing.
|
||||||
|
func TestAssigningTheClientIssuesItsAccount(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
client := aClientModule()
|
||||||
|
register(t, open, client)
|
||||||
|
out, err := assign(ctx, open, "laptop", client.Module)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "bus credential") {
|
||||||
|
t.Fatalf("assigning the client said nothing of its account:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+20
-17
@@ -180,12 +180,16 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// And build results nobody was waiting for. A build triggered any other way than `build`
|
// And build results nobody was waiting for. A build triggered any other way than `build`
|
||||||
// would otherwise be reported into the void, which is the same as not reporting it.
|
// would otherwise be reported into the void, which is the same as not reporting it.
|
||||||
server.Records(builds{inv, open})
|
server.Records(builds{inv, open})
|
||||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
// Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the
|
||||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
// tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left
|
||||||
go planTicker(ctx, open)
|
// by a replaced controller resumes from the first look, which this asks for. No timer looks every so often.
|
||||||
// And the merge window (novox/hq ADR 0276): a batch is cut into its walk when the window closes, which no
|
startRolloutEvents(ctx, open, server.JetStream())
|
||||||
// merge or outcome says.
|
// And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes,
|
||||||
go batchCutter(ctx, open)
|
// which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it;
|
||||||
|
// nothing polls (issue 973).
|
||||||
|
if err := startMergeWindow(ctx, open, server.JetStream()); err != nil {
|
||||||
|
return fmt.Errorf("the merge window could not be put on the bus: %w", err)
|
||||||
|
}
|
||||||
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
||||||
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
||||||
// Never by a read.
|
// Never by a read.
|
||||||
@@ -198,9 +202,8 @@ func serve(ctx context.Context) (err error) {
|
|||||||
if err := server.Follows(following{open: open}); err != nil {
|
if err := server.Follows(following{open: open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
// No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable
|
||||||
// rather than never (novox/hq issue 266).
|
// consumer on the bus keep every merge, and one the controller cannot record yet is held and handed over again.
|
||||||
go catchingUpOnMerges(ctx, open, server)
|
|
||||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||||
// both: what a build meant and what the builds were are two questions about one record.
|
// both: what a build meant and what the builds were are two questions about one record.
|
||||||
if err := server.Answers(following{open: open}); err != nil {
|
if err := server.Answers(following{open: open}); err != nil {
|
||||||
@@ -250,13 +253,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// (novox/hq to-be 45 §9).
|
// (novox/hq to-be 45 §9).
|
||||||
checkEvents = bus
|
checkEvents = bus
|
||||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
|
inventory.PlanSaved = servingPlanSaved(ctx, bus, open)
|
||||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
|
||||||
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
|
|
||||||
if p.WalkSaid == inventory.WalkEnded {
|
|
||||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := server.Checks(following{open: open}); err != nil {
|
if err := server.Checks(following{open: open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1333,8 +1330,14 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
}
|
}
|
||||||
// Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the
|
// Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the
|
||||||
// router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement.
|
// router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement.
|
||||||
records.RootFree = rootFreeNow(ctx, newRootReader(ctx, open.inventory, bus.Conn), records.Nodes, time.Now())
|
var whys map[string]string
|
||||||
|
records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn),
|
||||||
|
records.Nodes, time.Now()))
|
||||||
where := broker.PlacementsOf(records, records.Interchangeable)
|
where := broker.PlacementsOf(records, records.Interchangeable)
|
||||||
|
// What the judgement kept off is said, with its reason (novox/hq issue 930); the self-check gives it back.
|
||||||
|
for _, line := range droppedVerifiedSenders(records, where, whys) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
||||||
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
||||||
// its bucket only after the control plane next restarts — found the first time a module declared
|
// its bucket only after the control plane next restarts — found the first time a module declared
|
||||||
|
|||||||
@@ -80,30 +80,6 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
|
|
||||||
// the one process that puts the queue in order, so no two passes pause and resume it over each other.
|
|
||||||
// The build seat is read from the catalogue at most every half minute.
|
|
||||||
func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
|
|
||||||
if js == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
seat, readAt := link.TheBuildMachine, time.Time{}
|
|
||||||
seatNow := func() string {
|
|
||||||
if time.Since(readAt) < watchEvery {
|
|
||||||
return seat
|
|
||||||
}
|
|
||||||
if entries, err := open.inventory.Catalogued(ctx); err == nil {
|
|
||||||
seat, readAt = buildSeatAmong(entries), time.Now()
|
|
||||||
} else {
|
|
||||||
fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
|
|
||||||
}
|
|
||||||
return seat
|
|
||||||
}
|
|
||||||
link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
|
|
||||||
fmt.Printf(format+"\n", args...)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
||||||
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
||||||
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
||||||
@@ -303,16 +279,8 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
|||||||
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
// The worker of the queue it waits in: the seat's own, or one of the two check queues' (novox/hq ADR 0406 §3).
|
||||||
// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
|
worker := link.WorkerOf(seat, ask)
|
||||||
// later sequence, and the worker handing out what was behind its old one is not it being taken.
|
|
||||||
if ask.State == link.AskWaiting {
|
|
||||||
if q, err := link.ReadQueue(ctx, js, seat); err == nil {
|
|
||||||
if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
|
|
||||||
ask.Seq = now.Seq
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch ask.State {
|
switch ask.State {
|
||||||
case link.AskWaiting:
|
case link.AskWaiting:
|
||||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||||
@@ -334,7 +302,6 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
|||||||
"ends it there", ask.ID, started, ask.ID)
|
"ends it there", ask.ID, started, ask.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Found by its id when it was moved into order since it was read (novox/hq issue 558).
|
|
||||||
removed, err := link.DeleteAsk(ctx, js, seat, ask)
|
removed, err := link.DeleteAsk(ctx, js, seat, ask)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||||
|
|||||||
@@ -0,0 +1,224 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A channel's verified sender is composed into a membership only while the machines are judged root-free at
|
||||||
|
// that moment (novox/hq ADR 0259 §8), and the judgement fails closed when a read fails (novox/hq issue 930):
|
||||||
|
// a push composed during a failed read issued the router a membership without it, and nothing composed it
|
||||||
|
// again, so every approval was refused until somebody pushed. Two things follow.
|
||||||
|
//
|
||||||
|
// What a composition drops it says (droppedVerifiedSenders), with the reason the judgement gave.
|
||||||
|
//
|
||||||
|
// And the self-check issues it again (reissueVerifiedSenders): each time it judges the machines, a membership
|
||||||
|
// that lacks a verified sender the composition would now give it is issued again with only its kinds changed.
|
||||||
|
// **Bound: one self-check, doctorEvery (five minutes), after the judgement passes.** It only ever adds: a
|
||||||
|
// machine that stops passing keeps what it was issued, as at a push, because the router asks the root-free
|
||||||
|
// verb again before it honours an approval, and that is the check that holds.
|
||||||
|
|
||||||
|
// droppedVerifiedSenders says, one line each, every kind that declares `verified-sender` and was composed
|
||||||
|
// without it, naming the kind, its machine and why. whys is each machine's verdict reason (rootVerdict.Why).
|
||||||
|
func droppedVerifiedSenders(r broker.Records, where broker.Placements, whys map[string]string) []string {
|
||||||
|
var routerNodes []string
|
||||||
|
for node, declared := range r.Assigned {
|
||||||
|
for _, d := range declared {
|
||||||
|
for _, s := range d.Holds {
|
||||||
|
if s.Name == routerSeat && !slices.Contains(routerNodes, node) {
|
||||||
|
routerNodes = append(routerNodes, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(routerNodes)
|
||||||
|
var out []string
|
||||||
|
for _, k := range where.Kinds {
|
||||||
|
declared := false
|
||||||
|
for _, d := range r.Assigned[k.Node] {
|
||||||
|
for _, s := range d.Holds {
|
||||||
|
if d.Module == k.Module && s.Name == k.Seat && s.Kind == k.Kind && slices.Contains(s.Capabilities, "verified-sender") {
|
||||||
|
declared = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !declared || slices.Contains(k.Capabilities, "verified-sender") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var why string
|
||||||
|
switch {
|
||||||
|
case !r.RootFree[k.Node]:
|
||||||
|
why = k.Node + " is not judged root-free: " + whys[k.Node]
|
||||||
|
case len(routerNodes) != 1:
|
||||||
|
why = fmt.Sprintf("the router is placed on %d machines, not one", len(routerNodes))
|
||||||
|
case !r.RootFree[routerNodes[0]]:
|
||||||
|
why = "the router's machine " + routerNodes[0] + " is not judged root-free: " + whys[routerNodes[0]]
|
||||||
|
default:
|
||||||
|
why = k.Module + " does not run as an account of its own"
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf(" %s kind %s of %s on %s is composed without its verified sender: %s",
|
||||||
|
k.Seat, k.Kind, k.Module, k.Node, why))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withVerifiedSenders is a membership as issued with its seat traffic's kinds replaced by the composed ones,
|
||||||
|
// and whether that changed it: only when a composed kind carries a verified sender the issued one lacks. Pure.
|
||||||
|
func withVerifiedSenders(issued []byte, composed *broker.SeatTraffic) ([]byte, bool, error) {
|
||||||
|
if composed == nil {
|
||||||
|
return issued, false, nil
|
||||||
|
}
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(issued, &fields); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
var traffic map[string]json.RawMessage
|
||||||
|
var had []broker.KindHeld
|
||||||
|
if raw, ok := fields["seat-traffic"]; ok {
|
||||||
|
if err := json.Unmarshal(raw, &traffic); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if raw, ok := traffic["kinds"]; ok {
|
||||||
|
if err := json.Unmarshal(raw, &had); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gains := false
|
||||||
|
for _, k := range composed.Kinds {
|
||||||
|
if !slices.Contains(k.Capabilities, "verified-sender") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
held := false
|
||||||
|
for _, h := range had {
|
||||||
|
if h.Seat == k.Seat && h.Kind == k.Kind && h.Module == k.Module && h.Node == k.Node &&
|
||||||
|
slices.Contains(h.Capabilities, "verified-sender") {
|
||||||
|
held = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gains = gains || !held
|
||||||
|
}
|
||||||
|
if !gains {
|
||||||
|
return issued, false, nil
|
||||||
|
}
|
||||||
|
if traffic == nil {
|
||||||
|
traffic = map[string]json.RawMessage{}
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(composed.Kinds)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
traffic["kinds"] = raw
|
||||||
|
if fields["seat-traffic"], err = json.Marshal(traffic); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(fields)
|
||||||
|
return body, true, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// reissueVerifiedSenders issues again each membership that lacks a verified sender the records now give it:
|
||||||
|
// read as last issued, its kinds replaced, published only if no newer one was issued since (a push runs beside
|
||||||
|
// the self-check, and its newer membership must not be overwritten by the older fields read here). One never issued is left to its own send. Failures are
|
||||||
|
// said and passed over; the next self-check tries again.
|
||||||
|
func reissueVerifiedSenders(r broker.Records, where broker.Placements,
|
||||||
|
last func(node, module string) ([]byte, uint64, error), publish func(node, module string, body []byte, seq uint64) error,
|
||||||
|
say func(format string, args ...any)) int {
|
||||||
|
issued := 0
|
||||||
|
for node, declared := range r.Assigned {
|
||||||
|
for _, d := range declared {
|
||||||
|
composed := broker.MembershipFor(node, d, where).SeatTraffic
|
||||||
|
if composed == nil || !slices.ContainsFunc(composed.Kinds, func(k broker.KindHeld) bool {
|
||||||
|
return slices.Contains(k.Capabilities, "verified-sender")
|
||||||
|
}) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
was, seq, err := last(node, d.Module)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, nats.ErrMsgNotFound) {
|
||||||
|
say(" %s on %s: its membership could not be read to give its verified sender back: %v", d.Module, node, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, changed, err := withVerifiedSenders(was, composed)
|
||||||
|
if err != nil {
|
||||||
|
say(" %s on %s: its membership could not be read to give its verified sender back: %v", d.Module, node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !changed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := publish(node, d.Module, body, seq); err != nil {
|
||||||
|
// A push issued another since it was read: that one is newer, and the next self-check looks again.
|
||||||
|
if strings.Contains(err.Error(), "wrong last sequence") {
|
||||||
|
say(" %s on %s: a newer membership was issued meanwhile; its verified sender is looked at again next time",
|
||||||
|
d.Module, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
say(" %s on %s: its verified sender could not be issued again: %v", d.Module, node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
say(" %s on %s: issued its membership again, with the verified sender its machines now pass for", d.Module, node)
|
||||||
|
issued++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return issued
|
||||||
|
}
|
||||||
|
|
||||||
|
// rootVerdictsOf is each machine's verdict reason, for saying why one was not free.
|
||||||
|
func rootVerdictsOf(verdicts []rootVerdict) (free map[string]bool, whys map[string]string) {
|
||||||
|
free, whys = map[string]bool{}, map[string]string{}
|
||||||
|
for _, v := range verdicts {
|
||||||
|
if v.Free {
|
||||||
|
free[v.Machine] = true
|
||||||
|
}
|
||||||
|
whys[v.Machine] = strings.TrimSpace(v.Why)
|
||||||
|
}
|
||||||
|
return free, whys
|
||||||
|
}
|
||||||
|
|
||||||
|
// giveVerifiedSendersBack is the self-check's step: every machine judged, the memberships that lost a verified
|
||||||
|
// sender to a failed judgement issued again. Never fails the probe: what could not be done is said.
|
||||||
|
func giveVerifiedSendersBack(ctx context.Context, d *doctor) {
|
||||||
|
if d.js == nil || d.open == nil || d.open.inventory == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
records, err := d.open.inventory.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" the memberships could not be read to give a verified sender back: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Judged for every machine the records name, as a push does, not only those the self-check judged.
|
||||||
|
r := newRootReader(ctx, d.open.inventory, d.js.Conn())
|
||||||
|
all := judgeRootFree(ctx, r, records.Nodes, time.Now())
|
||||||
|
records.RootFree, _ = rootVerdictsOf(all)
|
||||||
|
where := broker.PlacementsOf(records, records.Interchangeable)
|
||||||
|
reissueVerifiedSendersOn(ctx, link.OverNATS{Conn: d.js.Conn(), JS: d.js.Context()}, records, where)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reissueVerifiedSendersOn is reissueVerifiedSenders on the bus: memberships read from the AssignmentsStream and
|
||||||
|
// published to it.
|
||||||
|
func reissueVerifiedSendersOn(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements) int {
|
||||||
|
return reissueVerifiedSenders(records, where,
|
||||||
|
func(node, module string) ([]byte, uint64, error) {
|
||||||
|
m, err := bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
return m.Data, m.Sequence, nil
|
||||||
|
},
|
||||||
|
func(node, module string, body []byte, seq uint64) error {
|
||||||
|
return bus.PublishMembershipAfter(ctx, node, module, body, seq)
|
||||||
|
},
|
||||||
|
func(format string, args ...any) { fmt.Printf(format+"\n", args...) })
|
||||||
|
}
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func verifiedSenderRecords(free map[string]bool) (broker.Records, broker.Declared) {
|
||||||
|
tg := broker.Seat{Name: "channel", Scope: "mesh", Accepts: []string{"show"}, Kinded: true, Kind: "telegram",
|
||||||
|
DeclaredBy: "messenger", Capabilities: []string{"choice", "verified-sender"}}
|
||||||
|
router := broker.Declared{Module: "messenger", RunsAs: "messenger", Holds: []broker.Seat{{Name: "operator-channel",
|
||||||
|
Scope: "mesh", Accepts: []string{"ask"}}}, Uses: []broker.Seat{{Name: "channel", Scope: "mesh", Kinded: true,
|
||||||
|
DeclaredBy: "messenger"}}}
|
||||||
|
return broker.Records{Nodes: []string{"novox"}, RootFree: free, Assigned: map[string][]broker.Declared{
|
||||||
|
"novox": {router, {Module: "telegram", RunsAs: "telegram", Holds: []broker.Seat{tg}}}}}, router
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeBus struct {
|
||||||
|
last map[string][]byte
|
||||||
|
published map[string][]byte
|
||||||
|
seq map[string]uint64
|
||||||
|
// between runs once, after a read and before the write: a push issuing meanwhile.
|
||||||
|
between func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeBus) get(node, module string) ([]byte, uint64, error) {
|
||||||
|
if b, ok := f.last[node+"/"+module]; ok {
|
||||||
|
return b, f.seq[node+"/"+module], nil
|
||||||
|
}
|
||||||
|
return nil, 0, nats.ErrMsgNotFound
|
||||||
|
}
|
||||||
|
func (f *fakeBus) put(node, module string, body []byte, seq uint64) error {
|
||||||
|
if f.between != nil {
|
||||||
|
f.between()
|
||||||
|
f.between = nil
|
||||||
|
}
|
||||||
|
if seq != f.seq[node+"/"+module] {
|
||||||
|
return errors.New("nats: wrong last sequence")
|
||||||
|
}
|
||||||
|
f.published[node+"/"+module] = body
|
||||||
|
f.last[node+"/"+module] = body
|
||||||
|
f.seq[node+"/"+module]++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasVerified(t *testing.T, membership []byte) bool {
|
||||||
|
t.Helper()
|
||||||
|
var m broker.Membership
|
||||||
|
if err := json.Unmarshal(membership, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, k := range m.SeatTraffic.Kinds {
|
||||||
|
if k.Kind == "telegram" {
|
||||||
|
return strings.Contains(strings.Join(k.Capabilities, ","), "verified-sender")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("telegram not in the router's membership")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// A membership composed while the judgement failed is issued again once it passes, with the verified sender
|
||||||
|
// (novox/hq issue 930); while it fails, nothing is given back (ADR 0259 §8).
|
||||||
|
func TestAMembershipThatLostItsVerifiedSenderGetsItBackWhenTheJudgementPasses(t *testing.T) {
|
||||||
|
failed, router := verifiedSenderRecords(nil)
|
||||||
|
whereFailed := broker.PlacementsOf(failed, nil)
|
||||||
|
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
|
||||||
|
body, _ := json.Marshal(broker.MembershipFor("novox", router, whereFailed))
|
||||||
|
bus.last["novox/messenger"] = body
|
||||||
|
if hasVerified(t, body) {
|
||||||
|
t.Fatal("composed with a failed judgement, and the verified sender is kept")
|
||||||
|
}
|
||||||
|
say := func(string, ...any) {}
|
||||||
|
|
||||||
|
if n := reissueVerifiedSenders(failed, whereFailed, bus.get, bus.put, say); n != 0 || len(bus.published) != 0 {
|
||||||
|
t.Fatalf("issued %d while the judgement still fails", n)
|
||||||
|
}
|
||||||
|
passing, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||||
|
wherePassing := broker.PlacementsOf(passing, nil)
|
||||||
|
if n := reissueVerifiedSenders(passing, wherePassing, bus.get, bus.put, say); n != 1 {
|
||||||
|
t.Fatalf("issued %d memberships after the judgement passed, want 1", n)
|
||||||
|
}
|
||||||
|
if !hasVerified(t, bus.published["novox/messenger"]) {
|
||||||
|
t.Fatal("the membership issued again lacks the verified sender")
|
||||||
|
}
|
||||||
|
if n := reissueVerifiedSenders(passing, wherePassing, bus.get, bus.put, say); n != 0 {
|
||||||
|
t.Fatalf("issued %d again with nothing changed", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a composition drops it says, naming the kind, the machine and the judgement's reason.
|
||||||
|
func TestADroppedVerifiedSenderIsSaidWithItsReason(t *testing.T) {
|
||||||
|
records, _ := verifiedSenderRecords(nil)
|
||||||
|
where := broker.PlacementsOf(records, nil)
|
||||||
|
lines := droppedVerifiedSenders(records, where, map[string]string{"novox": "not measured: no verb heard"})
|
||||||
|
if len(lines) != 1 || !strings.Contains(lines[0], "telegram") || !strings.Contains(lines[0], "novox") ||
|
||||||
|
!strings.Contains(lines[0], "not measured: no verb heard") {
|
||||||
|
t.Fatalf("said %q", lines)
|
||||||
|
}
|
||||||
|
records, _ = verifiedSenderRecords(map[string]bool{"novox": true})
|
||||||
|
if lines := droppedVerifiedSenders(records, broker.PlacementsOf(records, nil), nil); len(lines) != 0 {
|
||||||
|
t.Fatalf("said %q of a kind that kept its verified sender", lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMembershipNeverIssuedIsLeftToItsOwnSend(t *testing.T) {
|
||||||
|
records, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||||
|
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
|
||||||
|
var said []string
|
||||||
|
n := reissueVerifiedSenders(records, broker.PlacementsOf(records, nil), bus.get, bus.put,
|
||||||
|
func(f string, a ...any) { said = append(said, f) })
|
||||||
|
if n != 0 || len(said) != 0 {
|
||||||
|
t.Fatalf("issued %d, said %v", n, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **On a real bus** (novox/hq issue 930): a router's membership issued without the verified sender because the
|
||||||
|
// judgement failed is, once the judgement passes, published again to the AssignmentsStream with it in its seat
|
||||||
|
// traffic's kinds, and a second pass publishes nothing.
|
||||||
|
func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgementPasses(t *testing.T) {
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()}
|
||||||
|
|
||||||
|
failed, router := verifiedSenderRecords(nil)
|
||||||
|
body, err := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := bus.PublishMembership(t.Context(), "novox", "messenger", body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
issued := func() []byte {
|
||||||
|
m, err := js.Context().GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject("novox", "messenger"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return m.Data
|
||||||
|
}
|
||||||
|
if hasVerified(t, issued()) {
|
||||||
|
t.Fatal("issued with a failed judgement, and the verified sender is kept: the test proves nothing")
|
||||||
|
}
|
||||||
|
if n := reissueVerifiedSendersOn(t.Context(), bus, failed, broker.PlacementsOf(failed, nil)); n != 0 {
|
||||||
|
t.Fatalf("issued %d memberships while the judgement still fails", n)
|
||||||
|
}
|
||||||
|
passing, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||||
|
where := broker.PlacementsOf(passing, nil)
|
||||||
|
if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 1 {
|
||||||
|
t.Fatalf("issued %d memberships after the judgement passed, want 1", n)
|
||||||
|
}
|
||||||
|
if !hasVerified(t, issued()) {
|
||||||
|
t.Fatal("the membership on the bus lacks the verified sender after the judgement passed")
|
||||||
|
}
|
||||||
|
if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 0 {
|
||||||
|
t.Fatalf("a second pass issued %d memberships", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A push issuing a newer membership between the self-check's read and its write is not overwritten (issue 930).
|
||||||
|
func TestANewerMembershipIssuedMeanwhileIsNotOverwritten(t *testing.T) {
|
||||||
|
failed, router := verifiedSenderRecords(nil)
|
||||||
|
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
|
||||||
|
old, _ := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil)))
|
||||||
|
bus.last["novox/messenger"], bus.seq["novox/messenger"] = old, 1
|
||||||
|
newer := []byte(`{"node":"novox","module":"messenger","newer":true}`)
|
||||||
|
bus.between = func() { bus.last["novox/messenger"], bus.seq["novox/messenger"] = newer, 2 }
|
||||||
|
passing, _ := verifiedSenderRecords(map[string]bool{"novox": true})
|
||||||
|
if n := reissueVerifiedSenders(passing, broker.PlacementsOf(passing, nil), bus.get, bus.put, func(string, ...any) {}); n != 0 {
|
||||||
|
t.Fatalf("issued %d over a newer membership", n)
|
||||||
|
}
|
||||||
|
if string(bus.last["novox/messenger"]) != string(newer) || len(bus.published) != 0 {
|
||||||
|
t.Fatalf("the newer membership was overwritten: %s", bus.last["novox/messenger"])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,10 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
@@ -11,7 +13,9 @@ import (
|
|||||||
// related when they share a repository, a module, a delivery group's head, or an edge of the module graph
|
// related when they share a repository, a module, a delivery group's head, or an edge of the module graph
|
||||||
// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the
|
// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the
|
||||||
// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each
|
// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each
|
||||||
// other (ADR 0304 decision 2, ADR 0282 decision 3); nothing walks so yet, and nothing but tests reads this.
|
// other (ADR 0304 decision 2, ADR 0282 decision 3); a closed batch is cut beside an open walk it is
|
||||||
|
// not related to, and a walk about to ask its first tier waits only for a started walk it is related to (issue 973,
|
||||||
|
// ADR 0407 decision 4).
|
||||||
|
|
||||||
// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group
|
// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group
|
||||||
// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it
|
// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it
|
||||||
@@ -109,3 +113,83 @@ func firstShared(a, b map[string]bool) string {
|
|||||||
sort.Strings(both)
|
sort.Strings(both)
|
||||||
return both[0]
|
return both[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// openWalk is a walk open beside a change: its id, what it reaches, and whether it is on the controller's own path.
|
||||||
|
type openWalk struct {
|
||||||
|
id string
|
||||||
|
fp footprint
|
||||||
|
own bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// relatedOpen is the first open walk a change is related to, with the reason; empty when it is related to none
|
||||||
|
// and may walk beside them all (issue 973, ADR 0407 decision 4, ADR 0306 decision 3).
|
||||||
|
//
|
||||||
|
// **The controller's own path is related to everything** (decided for issue 973): a walk or batch that replaces
|
||||||
|
// the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside another, and
|
||||||
|
// nothing runs beside it, whatever the footprints say.
|
||||||
|
func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventory.Edge) (behind, why string) {
|
||||||
|
for _, w := range open {
|
||||||
|
if candOwn {
|
||||||
|
return w.id, "it is on the controller's own path"
|
||||||
|
}
|
||||||
|
if w.own {
|
||||||
|
return w.id, w.id + " is on the controller's own path"
|
||||||
|
}
|
||||||
|
if ok, why := related(cand, w.fp, edges); ok {
|
||||||
|
return w.id, why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path
|
||||||
|
// (issue 973).
|
||||||
|
func ownWalk(p inventory.Plan) bool {
|
||||||
|
if p.OwnPath() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for name := range p.Modules {
|
||||||
|
if _, own := onTheControllersPath[name]; own {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// startedWalks are the open walks started beside the walk id, made at created: a merge's walk past its wait, not
|
||||||
|
// the backlog's, with a tier asked, or on its way (let go, or waiting for nobody) before this one.
|
||||||
|
func startedWalks(plans []inventory.Plan, id string, created time.Time) []inventory.Plan {
|
||||||
|
var out []inventory.Plan
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == id || q.Release != nil || q.Waiting() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if q.Tier > 0 || askedAny(q) || q.Created.Before(created) {
|
||||||
|
out = append(out, q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the
|
||||||
|
// merges the record holds for it.
|
||||||
|
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (openWalk, error) {
|
||||||
|
kept, err := inv.MergesOf(ctx, p.ID)
|
||||||
|
if err != nil {
|
||||||
|
return openWalk{}, err
|
||||||
|
}
|
||||||
|
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// openWalksOf are openWalkOf for each of plans.
|
||||||
|
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) ([]openWalk, error) {
|
||||||
|
var out []openWalk
|
||||||
|
for _, p := range plans {
|
||||||
|
w, err := openWalkOf(ctx, inv, p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, w)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,8 +2,11 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -108,3 +111,125 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
|||||||
t.Fatalf("a known, unrelated batch read as related: %q", why)
|
t.Fatalf("a known, unrelated batch read as related: %q", why)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo: a change that shares nothing with any open walk is
|
||||||
|
// held behind none (issue 973, ADR 0407 decision 4); one that shares a repository, a module, a graph edge or a
|
||||||
|
// group head is held behind the first walk it shares it with, with the reason; one whose moves are not known
|
||||||
|
// is held behind the first; a walk or change on the controller's own path is related to every other.
|
||||||
|
func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) {
|
||||||
|
edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}}
|
||||||
|
walk := func(id, repo string, own bool, modules ...string) openWalk {
|
||||||
|
return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own}
|
||||||
|
}
|
||||||
|
one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget")
|
||||||
|
cand := func(repo, head string, moves ...string) footprint {
|
||||||
|
return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges)
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
fp footprint
|
||||||
|
own bool
|
||||||
|
open []openWalk
|
||||||
|
behind string
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""},
|
||||||
|
{"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""},
|
||||||
|
{"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"},
|
||||||
|
{"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two},
|
||||||
|
"plan-2", "both move nzbget"},
|
||||||
|
{"a graph edge", cand("novox/x", "fix/a", "nzbget"),
|
||||||
|
false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3",
|
||||||
|
"nzbget depends on base-image (stands-on)"},
|
||||||
|
{"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1",
|
||||||
|
"the modules a merge of novox/x moves are not known until its batch is cut"},
|
||||||
|
{"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1",
|
||||||
|
"it is on the controller's own path"},
|
||||||
|
{"an open walk on the own path", cand("novox/x", "fix/a", "app"), false,
|
||||||
|
[]openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4",
|
||||||
|
"plan-4 is on the controller's own path"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
behind, why := relatedOpen(c.fp, c.own, c.open, edges)
|
||||||
|
if behind != c.behind || why != c.why {
|
||||||
|
t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973).
|
||||||
|
func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) {
|
||||||
|
if ownWalk(walkOf("novox/one", "notes")) {
|
||||||
|
t.Error("an ordinary walk read as on the controller's own path")
|
||||||
|
}
|
||||||
|
flagged := walkOf("novox/one", "notes")
|
||||||
|
flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||||
|
if !ownWalk(flagged) {
|
||||||
|
t.Error("a walk whose batch is own-path read as ordinary")
|
||||||
|
}
|
||||||
|
var module string
|
||||||
|
for m := range onTheControllersPath {
|
||||||
|
module = m
|
||||||
|
}
|
||||||
|
if !ownWalk(walkOf("novox/one", "notes", module)) {
|
||||||
|
t.Errorf("a walk planning %s read as ordinary", module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk about to ask its first tier waits only for a started walk it is related to (issue 973): started is a
|
||||||
|
// tier asked, or a walk made before it; a waiting walk, the backlog's and itself are not.
|
||||||
|
func TestWalksStartedBesideAreTheOpenOnesBeforeIt(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||||
|
mk := func(id string, created time.Time, tier int) inventory.Plan {
|
||||||
|
p := walkOf("novox/"+id, id)
|
||||||
|
p.ID, p.Created, p.Tier, p.State = id, created, tier, inventory.PlanBuilding
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
older, newer, ticking := mk("older", at, 0), mk("newer", at.Add(time.Minute), 0), mk("ticking", at.Add(2*time.Minute), 1)
|
||||||
|
backlog := mk("backlog", at, 1)
|
||||||
|
backlog.Release = &inventory.PlanRelease{}
|
||||||
|
got := startedWalks([]inventory.Plan{older, newer, ticking, backlog, mk("me", at.Add(time.Minute), 0)}, "me", at.Add(time.Minute))
|
||||||
|
var ids []string
|
||||||
|
for _, p := range got {
|
||||||
|
ids = append(ids, p.ID)
|
||||||
|
}
|
||||||
|
if strings.Join(ids, ",") != "older,ticking" {
|
||||||
|
t.Errorf("started beside me = %v; want older, ticking", ids)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A retry is refused beside an open walk it is related to and allowed beside one it shares nothing with
|
||||||
|
// (issue 973); a retry on the controller's own path is refused beside any open walk.
|
||||||
|
func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) {
|
||||||
|
failed := walkOf("novox/one", "notes")
|
||||||
|
failed.ID, failed.State = "plan-1", inventory.PlanFailed
|
||||||
|
open := func(id, repo string, modules ...string) inventory.Plan {
|
||||||
|
p := walkOf(repo, modules...)
|
||||||
|
p.ID, p.State = id, inventory.PlanBuilding
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err != nil {
|
||||||
|
t.Errorf("refused beside an unrelated walk: %v", err)
|
||||||
|
}
|
||||||
|
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") {
|
||||||
|
t.Errorf("a retry beside a walk moving the same module: %v", err)
|
||||||
|
}
|
||||||
|
failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||||
|
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err == nil {
|
||||||
|
t.Error("an own-path retry went beside an open walk")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plans another controller is working are "not now", not a failure: the merge is held and handed over again
|
||||||
|
// (issue 921).
|
||||||
|
func TestPlansHeldByAnotherControllerAreNotNow(t *testing.T) {
|
||||||
|
if err := notNow(fmt.Errorf("x: %w", inventory.ErrPlansBusy)); !errors.Is(err, link.ErrTryAgain) {
|
||||||
|
t.Errorf("busy plans read as %v; want ErrTryAgain", err)
|
||||||
|
}
|
||||||
|
if err := notNow(errors.New("no such module")); errors.Is(err, link.ErrTryAgain) {
|
||||||
|
t.Errorf("another failure read as ErrTryAgain: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1023,12 +1023,12 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
|
|||||||
outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx))
|
outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx))
|
||||||
}
|
}
|
||||||
if !reported {
|
if !reported {
|
||||||
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
if now.Sub(*g.Since) > applySilentAfter {
|
||||||
g.Silent = true
|
g.Silent = true
|
||||||
r.Silent = append(r.Silent, node)
|
r.Silent = append(r.Silent, node)
|
||||||
changed = true
|
changed = true
|
||||||
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||||
inventory.ApplySilentAfter))
|
applySilentAfter))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
judging = append(judging, node+": not yet reported on its send")
|
judging = append(judging, node+": not yet reported on its send")
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ import (
|
|||||||
//
|
//
|
||||||
// The handler that hears the merge computes the plan from the catalogue's one dependency relation,
|
// The handler that hears the merge computes the plan from the catalogue's one dependency relation,
|
||||||
// writes it to the store, asks the first tier and returns — the receive loop is never held by a
|
// writes it to the store, asks the first tier and returns — the receive loop is never held by a
|
||||||
// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes
|
// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can
|
||||||
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report);
|
||||||
// plan where it left it.
|
// a controller replaced mid-plan finds the plan where it left it and looks first.
|
||||||
|
|
||||||
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
||||||
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
||||||
@@ -618,23 +618,32 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
|
|||||||
advanceHeld(ctx, open)
|
advanceHeld(ctx, open)
|
||||||
}
|
}
|
||||||
|
|
||||||
// startedBeside is the id of a started walk open beside this one — a merge's walk past its wait, not the
|
// startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past
|
||||||
// backlog's — or empty: one walk at a time (novox/hq ADR 0276).
|
// its wait, not the backlog's — with why, or empty: a walk waits only for a started walk it shares something with
|
||||||
func startedBeside(ctx context.Context, inv *inventory.Inventory, id string, created time.Time) (string, error) {
|
// (novox/hq ADR 0276 decision 4, ADR 0407 decision 4, issue 973).
|
||||||
|
func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (behind, why string, err error) {
|
||||||
plans, err := inv.OpenPlans(ctx)
|
plans, err := inv.OpenPlans(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
for _, q := range plans {
|
started := startedWalks(plans, p.ID, p.Created)
|
||||||
if q.ID == id || q.Release != nil || q.Waiting() {
|
if len(started) == 0 {
|
||||||
continue
|
return "", "", nil
|
||||||
}
|
|
||||||
// Started: a tier asked, or on its way (let go, or waiting for nobody) before this one.
|
|
||||||
if q.Tier > 0 || askedAny(q) || q.Created.Before(created) {
|
|
||||||
return q.ID, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return "", nil
|
edges, err := inv.Dependencies(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
open, err := openWalksOf(ctx, inv, started)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
mine, err := openWalkOf(ctx, inv, p)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
behind, why = relatedOpen(mine.fp, mine.own, open, edges)
|
||||||
|
return behind, why, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// deferredNote begins the note of a walk deferred behind another.
|
// deferredNote begins the note of a walk deferred behind another.
|
||||||
@@ -659,29 +668,6 @@ func askedAny(p inventory.Plan) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
|
|
||||||
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
|
|
||||||
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
|
|
||||||
//
|
|
||||||
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
|
|
||||||
// controllers — the old and the new while a machine hands its controller over — would each ask a
|
|
||||||
// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
|
|
||||||
func advancePlans(ctx context.Context, open *stores) {
|
|
||||||
release, err := open.inventory.HoldPlans(ctx, false)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, inventory.ErrPlansBusy) {
|
|
||||||
fmt.Printf("plans: cannot hold them: %v\n", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
|
||||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
|
||||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
|
||||||
}
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
|
||||||
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
|
||||||
func keepWalkPhases(ctx context.Context, open *stores) {
|
func keepWalkPhases(ctx context.Context, open *stores) {
|
||||||
@@ -690,10 +676,15 @@ func keepWalkPhases(ctx context.Context, open *stores) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules
|
||||||
func advanceHeld(ctx context.Context, open *stores) {
|
// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says
|
||||||
|
// whether a step failed, which the look at the plans (lookAtPlans) retries.
|
||||||
|
//
|
||||||
|
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the
|
||||||
|
// old and the new while a machine hands its controller over — would each ask a tier the other had just asked.
|
||||||
|
func advanceHeld(ctx context.Context, open *stores) (failed bool) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
// A walk that ended lets the next batch be cut at once, not at the cutter's next look (novox/hq ADR 0276).
|
// A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407).
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
||||||
fmt.Printf("batches: %v\n", err)
|
fmt.Printf("batches: %v\n", err)
|
||||||
@@ -702,15 +693,15 @@ func advanceHeld(ctx context.Context, open *stores) {
|
|||||||
plans, err := inv.OpenPlans(ctx)
|
plans, err := inv.OpenPlans(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("plans: cannot read them: %v\n", err)
|
fmt.Printf("plans: cannot read them: %v\n", err)
|
||||||
return
|
return true
|
||||||
}
|
}
|
||||||
if len(plans) == 0 {
|
if len(plans) == 0 {
|
||||||
return
|
return false
|
||||||
}
|
}
|
||||||
edges, err := inv.Dependencies(ctx)
|
edges, err := inv.Dependencies(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("plans: cannot read the dependencies: %v\n", err)
|
fmt.Printf("plans: cannot read the dependencies: %v\n", err)
|
||||||
return
|
return true
|
||||||
}
|
}
|
||||||
rollsOut := func(module string) bool {
|
rollsOut := func(module string) bool {
|
||||||
u, err := inv.UpgradeOf(ctx, module)
|
u, err := inv.UpgradeOf(ctx, module)
|
||||||
@@ -724,10 +715,12 @@ func advanceHeld(ctx context.Context, open *stores) {
|
|||||||
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
|
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
|
||||||
// said on the bus, and its last save read as when it began building.
|
// said on the bus, and its last save read as when it began building.
|
||||||
before := planSnapshot(*p)
|
before := planSnapshot(*p)
|
||||||
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
sends := &sendsMade{}
|
||||||
|
moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
failed = true
|
||||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||||
// the state is left as it was and the step is tried again on the next tick. Said and
|
// the state is left as it was and the step is tried again by the next look. Said and
|
||||||
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
||||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||||
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
|
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
|
||||||
@@ -748,14 +741,18 @@ func advanceHeld(ctx context.Context, open *stores) {
|
|||||||
if planSnapshot(*p) != before {
|
if planSnapshot(*p) != before {
|
||||||
if err := inv.SavePlan(ctx, p); err != nil {
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
|
failed = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
// Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent.
|
||||||
|
saySends(ctx, *p, sends)
|
||||||
}
|
}
|
||||||
if !moved {
|
if !moved {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return failed
|
||||||
}
|
}
|
||||||
|
|
||||||
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
|
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
|
||||||
@@ -793,13 +790,14 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
tier := p.Tiers[p.Tier]
|
tier := p.Tiers[p.Tier]
|
||||||
// Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754).
|
// Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754).
|
||||||
if unasked := unaskedOf(*p); len(unasked) > 0 {
|
if unasked := unaskedOf(*p); len(unasked) > 0 {
|
||||||
// **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started
|
// **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): a walk
|
||||||
// walk is open waits for that one to end, let go or not, and says so.
|
// about to ask its first tier while a started walk it is related to is open waits for that one to end,
|
||||||
|
// let go or not, and says so and why. A started walk it shares nothing with does not hold it.
|
||||||
if p.Tier == 0 && !askedInTier(*p) {
|
if p.Tier == 0 && !askedInTier(*p) {
|
||||||
if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil {
|
if behind, why, err := startedBeside(ctx, inv, *p); err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
} else if behind != "" {
|
} else if behind != "" {
|
||||||
note := fmt.Sprintf("%s%s ended — one walk at a time", deferredNote, behind)
|
note := fmt.Sprintf("%s%s ended — %s", deferredNote, behind, why)
|
||||||
changed := p.Note != note
|
changed := p.Note != note
|
||||||
p.Note = note
|
p.Note = note
|
||||||
return changed, nil
|
return changed, nil
|
||||||
@@ -1056,6 +1054,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
p.Modules[m].SentAt = &now
|
p.Modules[m].SentAt = &now
|
||||||
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
|
||||||
}
|
}
|
||||||
|
// One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974).
|
||||||
|
for _, n := range sent {
|
||||||
|
var carried []string
|
||||||
|
for _, m := range rest {
|
||||||
|
if slices.Contains(restTo[m], n) {
|
||||||
|
carried = append(carried, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
noteSends(ctx, p.Tier, []string{n}, carried, now)
|
||||||
|
}
|
||||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
return true, nil
|
return true, nil
|
||||||
@@ -1166,7 +1174,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
|
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
|
||||||
}
|
}
|
||||||
if len(silent) > 0 {
|
if len(silent) > 0 {
|
||||||
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter,
|
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter,
|
||||||
strings.Join(silent, ", ")))
|
strings.Join(silent, ", ")))
|
||||||
}
|
}
|
||||||
said = append(said, recordedSaid(*p)...)
|
said = append(said, recordedSaid(*p)...)
|
||||||
@@ -1231,7 +1239,7 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time,
|
|||||||
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
|
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
|
||||||
failed = append(failed, node+" ("+outcome+")")
|
failed = append(failed, node+" ("+outcome+")")
|
||||||
}
|
}
|
||||||
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
|
case !at.IsZero() && now.Sub(at) > applySilentAfter:
|
||||||
seen[node] = true
|
seen[node] = true
|
||||||
silent = append(silent, node)
|
silent = append(silent, node)
|
||||||
default:
|
default:
|
||||||
@@ -1328,6 +1336,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
s.GatedBy = lead
|
s.GatedBy = lead
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
noteSends(ctx, p.Tier, sent, modules, now)
|
||||||
p.State = inventory.PlanRolling
|
p.State = inventory.PlanRolling
|
||||||
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
|
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
|
||||||
strings.Join(sent, ", "))
|
strings.Join(sent, ", "))
|
||||||
@@ -1642,23 +1651,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
|||||||
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
|
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
|
||||||
}
|
}
|
||||||
|
|
||||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
|
||||||
func planTicker(ctx context.Context, open *stores) {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
keepWalkPhases(ctx, open)
|
|
||||||
tick := time.NewTicker(30 * time.Second)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
keepWalkPhases(ctx, open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// planLine is one plan as `status` says it, given the least bound of a tier.
|
// planLine is one plan as `status` says it, given the least bound of a tier.
|
||||||
func planLine(p inventory.Plan, now time.Time) string {
|
func planLine(p inventory.Plan, now time.Time) string {
|
||||||
return planLineWith(p, now, pauseView{}, tierAtLeast)
|
return planLineWith(p, now, pauseView{}, tierAtLeast)
|
||||||
@@ -1722,7 +1714,7 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
|||||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since)
|
return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since)
|
||||||
}
|
}
|
||||||
if deferred(p) {
|
if deferred(p) {
|
||||||
// Nor is waiting for the walk before it to end (one walk at a time, novox/hq ADR 0276).
|
// Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407).
|
||||||
return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
|
return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
|
||||||
}
|
}
|
||||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||||
@@ -2024,7 +2016,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
|
||||||
short(p.Commit))
|
short(p.Commit))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,345 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Acceptance of the step (novox/hq ADR 0406 §5, §6, acceptance 7; issue 974), live, with the store and a real bus and no
|
||||||
|
// ticker: a rollout of two tiers advances on the events alone — the plan kept, the machines' reports, a silent key's
|
||||||
|
// expiry — a machine that reports in time is never named silent and one that never reports is named silent when its
|
||||||
|
// key expires, and a controller replaced in the middle of it resumes with no tier asked twice and none skipped.
|
||||||
|
func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
// A quarter of an hour of silence is two seconds here.
|
||||||
|
wasSilent := applySilentAfter
|
||||||
|
applySilentAfter = 2 * time.Second
|
||||||
|
t.Cleanup(func() { applySilentAfter = wasSilent })
|
||||||
|
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||||
|
t.Cleanup(func() { checkEvents = nil })
|
||||||
|
sentHeard := make(chan *nats.Msg, 32)
|
||||||
|
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
|
||||||
|
// Every save of a plan is kept, said and looked at by the serving controller's own hook.
|
||||||
|
saved := &movesSaid{revs: map[string][]int64{}}
|
||||||
|
wasSaved := inventory.PlanSaved
|
||||||
|
serving := servingPlanSaved(ctx, checkEvents, open)
|
||||||
|
inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); serving(i, p) }
|
||||||
|
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||||
|
|
||||||
|
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||||
|
var mu sync.Mutex
|
||||||
|
var asks []string
|
||||||
|
var sends [][]string
|
||||||
|
wasAsk := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||||
|
asks = append(asks, source.Repository+"#"+id)
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = wasAsk })
|
||||||
|
digests := 0
|
||||||
|
wasSend := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
sends = append(sends, append([]string(nil), names...))
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
builds := map[string]string{}
|
||||||
|
for _, m := range []string{"a", "b"} {
|
||||||
|
builds[m] = current[m].Commit
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
digests++
|
||||||
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
|
|
||||||
|
register := func(m, commit string, asked time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||||
|
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||||
|
Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record := func(id, m, commit string, at time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||||
|
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
for _, m := range []string{"a", "b"} {
|
||||||
|
record("build-"+m+"-1", m, "c1", old)
|
||||||
|
register(m, "c1", old)
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||||
|
register("a", "c2", time.Now().Add(-time.Minute))
|
||||||
|
|
||||||
|
// A machine's report on what it was sent, heard as the controller hears it.
|
||||||
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||||
|
seq := int64(0)
|
||||||
|
reports := func(node string) {
|
||||||
|
t.Helper()
|
||||||
|
last, found, err := inv.SentTo(ctx, node)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||||
|
}
|
||||||
|
seq++
|
||||||
|
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||||
|
"report_sequence": seq})
|
||||||
|
var report link.Report
|
||||||
|
if err := json.Unmarshal(body, &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := listener.Heard(ctx, report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
heldKeys := func() map[silentWait]time.Time {
|
||||||
|
c := silentClockOf()
|
||||||
|
if c == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
held, err := c.Held(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
plan := func() inventory.Plan {
|
||||||
|
t.Helper()
|
||||||
|
p, err := inv.PlanByID(ctx, "plan-974")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
|
||||||
|
first, stopFirst := context.WithCancel(ctx)
|
||||||
|
defer stopFirst()
|
||||||
|
startRolloutEvents(first, open, js)
|
||||||
|
built := time.Now().UTC()
|
||||||
|
p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built,
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||||
|
"b": {},
|
||||||
|
}}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(sends) == 1 && len(asks) == 1 && plan().Tier == 1
|
||||||
|
})
|
||||||
|
eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 })
|
||||||
|
|
||||||
|
// The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody.
|
||||||
|
stopFirst()
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
reports("anchor")
|
||||||
|
|
||||||
|
second, stopSecond := context.WithCancel(ctx)
|
||||||
|
defer stopSecond()
|
||||||
|
startRolloutEvents(second, open, js)
|
||||||
|
// Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire.
|
||||||
|
eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool {
|
||||||
|
h := heldKeys()
|
||||||
|
_, anchor := h[silentWait{Plan: "plan-974", Tier: 0, Node: "anchor"}]
|
||||||
|
return !anchor
|
||||||
|
})
|
||||||
|
|
||||||
|
// tier 1's build comes in, as the build machine's outcome is taken in.
|
||||||
|
b := plan().Modules["b"]
|
||||||
|
if b == nil || b.Build == "" {
|
||||||
|
t.Fatalf("tier 1 was not asked: %+v", b)
|
||||||
|
}
|
||||||
|
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||||
|
record(b.Build, "b", "c2", askedAt)
|
||||||
|
register("b", "c2", askedAt)
|
||||||
|
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||||
|
eventually(t, "tier 1 sent to both machines", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(sends) == 2
|
||||||
|
})
|
||||||
|
reports("anchor")
|
||||||
|
|
||||||
|
// laptop never reports: its key expires and the new controller names it silent, which ends the rollout.
|
||||||
|
eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {
|
||||||
|
p := plan()
|
||||||
|
return p.State == inventory.PlanDone
|
||||||
|
})
|
||||||
|
p = plan()
|
||||||
|
if !strings.Contains(p.Note, "laptop") || strings.Contains(p.Note, "anchor") || !strings.Contains(p.Note, "said nothing") {
|
||||||
|
t.Errorf("the plan ended with the note %q; want laptop named silent and not anchor", p.Note)
|
||||||
|
}
|
||||||
|
eventually(t, "no key left on the bus", func() bool { return len(heldKeys()) == 0 })
|
||||||
|
|
||||||
|
// No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent
|
||||||
|
// once to both machines, and said as sent once for each.
|
||||||
|
mu.Lock()
|
||||||
|
if len(asks) != 1 || !strings.HasPrefix(asks[0], "novox/b#") {
|
||||||
|
t.Errorf("asks = %v; want tier 1's module asked once", asks)
|
||||||
|
}
|
||||||
|
if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" {
|
||||||
|
t.Errorf("sends = %v; want both machines sent each tier once", sends)
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
said := map[string]int{}
|
||||||
|
for deadline := time.After(5 * time.Second); len(said) < 4; {
|
||||||
|
select {
|
||||||
|
case m := <-sentHeard:
|
||||||
|
var body planSentBody
|
||||||
|
if err := json.Unmarshal(m.Data, &body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatalf("sent was said for %v; want tiers 0 and 1 to both machines", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
for len(sentHeard) > 0 {
|
||||||
|
var body planSentBody
|
||||||
|
_ = json.Unmarshal((<-sentHeard).Data, &body)
|
||||||
|
said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++
|
||||||
|
}
|
||||||
|
for k, n := range said {
|
||||||
|
if n != 1 {
|
||||||
|
t.Errorf("sent for %s was said %d times", k, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And every move of the plan was said as plan-moved.
|
||||||
|
saved.noUnsaid(t, open)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Acceptance of the verb path (issue 974, review 1), live, with the store and a bus: a walk the delivery's owner lets
|
||||||
|
// go by the verb `deliver` after the operator's approval is asked its first tier with no other event. The verb runs as a
|
||||||
|
// child process, whose saves the serving controller does not hear; the handler asks the look as the verb ends.
|
||||||
|
func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||||
|
t.Cleanup(func() { checkEvents = nil })
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cut := time.Now().UTC()
|
||||||
|
waiting := inventory.Plan{ID: "plan-974-deliver", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8",
|
||||||
|
Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat},
|
||||||
|
Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The serving controller, as serve starts it.
|
||||||
|
wasSaved := inventory.PlanSaved
|
||||||
|
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||||
|
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||||
|
started, stop := context.WithCancel(ctx)
|
||||||
|
defer stop()
|
||||||
|
startRolloutEvents(started, open, js)
|
||||||
|
time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing
|
||||||
|
|
||||||
|
// The verb as its child process runs it: the delivery command, whose saves are not the serving controller's.
|
||||||
|
wasRun := runTheVerb
|
||||||
|
runTheVerb = func(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
|
was := inventory.PlanSaved
|
||||||
|
inventory.PlanSaved = func(*inventory.Inventory, inventory.Plan) {}
|
||||||
|
defer func() { inventory.PlanSaved = was }()
|
||||||
|
return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"})
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { runTheVerb = wasRun })
|
||||||
|
handlers, _, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := handlers["deliver"](ctx, []byte(`{"plan":"`+waiting.ID+`","why":"its turn"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "the walk's first tier asked, with no event but the verb's end", func() bool {
|
||||||
|
p, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||||
|
return err == nil && p.Modules["app"] != nil && p.Modules["app"].Build != "" && len(*asked) == 1
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A rollout's wait for a machine is a key on the bus (novox/hq ADR 0406 §5, issue 974): each send to a machine puts
|
||||||
|
// a key `<plan>.<tier>.<node>` that lives as long as the machine may stay silent (ApplySilentAfter); the machine's
|
||||||
|
// report on that send deletes it, so only a missing report lets it expire; and the expiry, heard by a watcher,
|
||||||
|
// ends the wait for that machine, which the plan then names silent. No timer looks every so often whether a
|
||||||
|
// machine has reported.
|
||||||
|
|
||||||
|
// silentWait is one machine's wait on one send of a plan's tier.
|
||||||
|
type silentWait struct {
|
||||||
|
Plan string
|
||||||
|
Tier int
|
||||||
|
Node string
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentKey is a wait's key. A plan id may hold a dot, so it is read from the right (parseSilentKey).
|
||||||
|
func silentKey(w silentWait) string { return w.Plan + "." + strconv.Itoa(w.Tier) + "." + w.Node }
|
||||||
|
|
||||||
|
// parseSilentKey reads a key back: the node is the last token, the tier the one before, the plan what is left.
|
||||||
|
func parseSilentKey(key string) (silentWait, bool) {
|
||||||
|
last := strings.LastIndexByte(key, '.')
|
||||||
|
if last < 1 || last == len(key)-1 {
|
||||||
|
return silentWait{}, false
|
||||||
|
}
|
||||||
|
rest, node := key[:last], key[last+1:]
|
||||||
|
mid := strings.LastIndexByte(rest, '.')
|
||||||
|
if mid < 1 || mid == len(rest)-1 {
|
||||||
|
return silentWait{}, false
|
||||||
|
}
|
||||||
|
tier, err := strconv.Atoi(rest[mid+1:])
|
||||||
|
if err != nil || tier < 0 {
|
||||||
|
return silentWait{}, false
|
||||||
|
}
|
||||||
|
return silentWait{Plan: rest[:mid], Tier: tier, Node: node}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentClock is the bus's side of the wait for a machine's report.
|
||||||
|
type silentClock interface {
|
||||||
|
// Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was.
|
||||||
|
Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error
|
||||||
|
// Held is every wait that has a key now, with when its send was.
|
||||||
|
Held(ctx context.Context) (map[silentWait]time.Time, error)
|
||||||
|
// Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller.
|
||||||
|
Reported(ctx context.Context, w silentWait) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// busSilent is the clock on the controller's bucket.
|
||||||
|
type busSilent struct {
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBusSilent opens the bucket and starts the watcher, which names each wait whose key expired until the context
|
||||||
|
// ends. Only expiry wakes it: the marker of an expiry is a purge, and a key a report deleted, or one Arm made again,
|
||||||
|
// leaves a delete (the vendored client reads the server's MaxAge marker as a purge, kv.go:971).
|
||||||
|
func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(silentWait)) (*busSilent, error) {
|
||||||
|
kv, err := api.KeyValue(ctx, broker.RolloutSilentBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the rollout's silent keys' bucket: %w", err)
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("watching the rollout's silent keys: %w", err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer w.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case e, ok := <-w.Updates():
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if e == nil || e.Operation() != jetstream.KeyValuePurge {
|
||||||
|
continue // a put, or a delete: a report's, or a send made again
|
||||||
|
}
|
||||||
|
if wait, ok := parseSilentKey(e.Key()); ok && silent != nil {
|
||||||
|
silent(wait)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return &busSilent{kv: kv}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
|
||||||
|
key := silentKey(w)
|
||||||
|
// Made again, not changed: a key's time to live is set when it is made.
|
||||||
|
if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl)))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *busSilent) Reported(ctx context.Context, w silentWait) error {
|
||||||
|
if err := s.kv.Delete(ctx, silentKey(w)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// leastSecond rounds a time to live up to a whole second, at least one: the server rounds a key's time down to
|
||||||
|
// whole seconds, so a fraction would close the key before the time it stands for.
|
||||||
|
func leastSecond(d time.Duration) time.Duration {
|
||||||
|
if d < time.Second {
|
||||||
|
return time.Second
|
||||||
|
}
|
||||||
|
return (d + time.Second - 1) / time.Second * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) {
|
||||||
|
keys, err := s.kv.Keys(ctx)
|
||||||
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
held := map[silentWait]time.Time{}
|
||||||
|
for _, key := range keys {
|
||||||
|
w, ok := parseSilentKey(key)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := s.kv.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
at, _ := time.Parse(time.RFC3339Nano, string(e.Value()))
|
||||||
|
held[w] = at
|
||||||
|
}
|
||||||
|
return held, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire.
|
||||||
|
func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
|
||||||
|
t.Helper()
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(conn.Close)
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
heard := make(chan silentWait, 16)
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
t.Cleanup(stop)
|
||||||
|
s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return s, heard
|
||||||
|
}
|
||||||
|
|
||||||
|
func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case w := <-heard:
|
||||||
|
return w, true
|
||||||
|
case <-time.After(within):
|
||||||
|
return silentWait{}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5):
|
||||||
|
// the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur.
|
||||||
|
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
|
||||||
|
s, heard := silentOnABus(t)
|
||||||
|
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
||||||
|
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want {
|
||||||
|
t.Fatalf("heard %+v (%v); want %+v", got, ok, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry.
|
||||||
|
func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) {
|
||||||
|
s, heard := silentOnABus(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"}
|
||||||
|
if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.Reported(ctx, w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok {
|
||||||
|
t.Fatalf("%+v was named silent after it reported", got)
|
||||||
|
}
|
||||||
|
// Reporting twice, or on a send that put no key, is no error.
|
||||||
|
if err := s.Reported(ctx, w); err != nil {
|
||||||
|
t.Fatalf("a second report: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A send made again to a machine puts its key again with the full time, and only the last send's key can expire.
|
||||||
|
func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) {
|
||||||
|
s, heard := silentOnABus(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"}
|
||||||
|
if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(1200 * time.Millisecond)
|
||||||
|
if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok {
|
||||||
|
t.Fatalf("%+v expired at the first send's time, not the second's", got)
|
||||||
|
}
|
||||||
|
if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w {
|
||||||
|
t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) {
|
||||||
|
for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} {
|
||||||
|
got, ok := parseSilentKey(silentKey(w))
|
||||||
|
if !ok || got != w {
|
||||||
|
t.Errorf("%+v became %+v (%v)", w, got, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "plan", "plan.x.ace", "plan..ace", ".1.ace"} {
|
||||||
|
if _, ok := parseSilentKey(bad); ok {
|
||||||
|
t.Errorf("%q was read as a wait", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -413,10 +413,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if err := need("members"); err != nil {
|
if err := need("members"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
argv := []string{"delivery", "check", "--members", str("members")}
|
||||||
if g := str("group"); g != "" {
|
if g := str("group"); g != "" {
|
||||||
return []string{"delivery", "check", "--group", g, "--members", str("members")}, nil
|
argv = []string{"delivery", "check", "--group", g, "--members", str("members")}
|
||||||
}
|
}
|
||||||
return []string{"delivery", "check", "--members", str("members")}, nil
|
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
|
||||||
|
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
|
||||||
|
argv = append(argv, "--ready")
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "deliver":
|
case "deliver":
|
||||||
if err := need("plan"); err != nil {
|
if err := need("plan"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1398,12 +1403,24 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||||
link.Acknowledge(ctx)
|
link.Acknowledge(ctx)
|
||||||
}
|
}
|
||||||
return runVerb(ctx, argv)
|
answer, err := runTheVerb(ctx, argv)
|
||||||
|
// **A plan the verb moved is looked at as it ends** (novox/hq ADR 0406 §5, issue 974): the verb ran as a
|
||||||
|
// child process, whose saves this process does not hear, so `deliver` after an approval, a stop, a retry
|
||||||
|
// and a send from a verb would otherwise wait for an unrelated event. Even when it failed: it may have
|
||||||
|
// kept a plan before it did.
|
||||||
|
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||||
|
askToLook()
|
||||||
|
}
|
||||||
|
return answer, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return handlers, behind, nil
|
return handlers, behind, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runTheVerb is how the serving controller runs a verb: as a command of its own. A variable so a test of what follows a
|
||||||
|
// verb needs no child process.
|
||||||
|
var runTheVerb = runVerb
|
||||||
|
|
||||||
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||||
func actsOnAPlan(args map[string]any) bool {
|
func actsOnAPlan(args map[string]any) bool {
|
||||||
for _, act := range []string{"stop", "close", "retry"} {
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
|
|||||||
@@ -126,11 +126,6 @@ var signalsTable = []signalRow{
|
|||||||
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||||
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||||
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||||
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
|
||||||
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
|
||||||
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
|
||||||
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
|
||||||
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
|
||||||
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||||
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||||
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||||
@@ -459,18 +454,6 @@ func watchLoop(f *signalFacts) []conditions.Observation {
|
|||||||
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func watchMerges(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, m := range f.merges {
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
|
||||||
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
|
||||||
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
|
||||||
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
|
||||||
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func watchAsks(f *signalFacts) []conditions.Observation {
|
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||||
var out []conditions.Observation
|
var out []conditions.Observation
|
||||||
for _, a := range f.asks {
|
for _, a := range f.asks {
|
||||||
@@ -686,8 +669,10 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
|||||||
// controller, or a seat's holders.
|
// controller, or a seat's holders.
|
||||||
func consumerWho(stream, consumer string) string {
|
func consumerWho(stream, consumer string) string {
|
||||||
switch {
|
switch {
|
||||||
case consumer == broker.ControllerName:
|
case consumer == broker.ControllerName || consumer == broker.MergesConsumerName:
|
||||||
return "the controller"
|
return "the controller"
|
||||||
|
case broker.IsCheckStream(stream):
|
||||||
|
return "the holder of " + broker.TheBuildSeat
|
||||||
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
|
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
|
||||||
seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
|
seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
|
||||||
return "the holder of " + seat
|
return "the holder of " + seat
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ func calm(now time.Time) *signalFacts {
|
|||||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||||
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||||
mergesPassed: now.Add(-time.Minute),
|
|
||||||
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||||
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
||||||
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
||||||
@@ -72,12 +71,6 @@ var suppressions = map[string]suppression{
|
|||||||
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||||
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||||
},
|
},
|
||||||
"S5": {
|
|
||||||
inside: func(f *signalFacts) {},
|
|
||||||
past: func(f *signalFacts) {
|
|
||||||
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"S6": {
|
"S6": {
|
||||||
inside: func(f *signalFacts) {
|
inside: func(f *signalFacts) {
|
||||||
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
|||||||
@@ -210,6 +210,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
|||||||
if news {
|
if news {
|
||||||
l.summary.nudge()
|
l.summary.nudge()
|
||||||
}
|
}
|
||||||
|
// A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the
|
||||||
|
// machine, and the look is what deletes its key, knowing the report was on that send.
|
||||||
|
if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil {
|
||||||
|
askToLook()
|
||||||
|
}
|
||||||
// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
|
// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
|
||||||
// raised or cleared from it. Only from an account of the machine the store took.
|
// raised or cleared from it. Only from an account of the machine the store took.
|
||||||
if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
|
if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
|
||||||
|
|||||||
@@ -1,157 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A new push to a pull request ends the check of its old head (novox/hq issue 726).
|
|
||||||
//
|
|
||||||
// A check judges one head; once the pull request has another, its verdict answers nobody and its run holds a build
|
|
||||||
// machine for minutes the new head waits behind. So when a check is asked for pull request N of a repository at a
|
|
||||||
// head, **every check of that pull request at another head ends**: a waiting one is dropped from the queue as
|
|
||||||
// `cancel` drops it, one in flight is ended on its machine as `kill` ends it. Each is said in the log.
|
|
||||||
//
|
|
||||||
// **Never anything else**: a try (`try`, asked for no pull request), a walk's build (no check), a delivery group's
|
|
||||||
// composed check (it answers several pull requests), another pull request's check, a dead ask (left for `clear
|
|
||||||
// --dead`), and a check of the same head — a recheck of that head is not another head.
|
|
||||||
|
|
||||||
// **Only for the pull request's current head.** Announcements of a pull request are not ordered, one is heard again
|
|
||||||
// after a restart, and a recheck asks for the head it recorded: a check asked for a head the pull request no longer
|
|
||||||
// has must never end the check of the head it has. So the forge is asked first which head the pull request has; only
|
|
||||||
// when it names this one does anything end, and when it cannot be asked, or names another, nothing ends and that is
|
|
||||||
// said.
|
|
||||||
|
|
||||||
// forgePullHead asks the forge, through its module's tools on the bus, which commit a pull request's head is. A
|
|
||||||
// variable so a test needs no forge.
|
|
||||||
var forgePullHead = func(ctx context.Context, js *broker.JetStream, owner, repo string, number int) (string, error) {
|
|
||||||
raw, _ := json.Marshal(map[string]any{"path": fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, number)})
|
|
||||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return "", errors.New(answer.Error)
|
|
||||||
}
|
|
||||||
var pull struct {
|
|
||||||
Result struct {
|
|
||||||
Head struct {
|
|
||||||
SHA string `json:"sha"`
|
|
||||||
} `json:"head"`
|
|
||||||
} `json:"result"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(answer.Result, &pull); err != nil {
|
|
||||||
return "", fmt.Errorf("the forge's answer about the pull request cannot be read: %w", err)
|
|
||||||
}
|
|
||||||
if pull.Result.Head.SHA == "" {
|
|
||||||
return "", errors.New("the forge named no head for the pull request")
|
|
||||||
}
|
|
||||||
return pull.Result.Head.SHA, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// supersededChecks are the checks in the queue that a check of owner/repo#number at head ends: of the same pull
|
|
||||||
// request, at another head, waiting or in flight, and not the ask keep.
|
|
||||||
func supersededChecks(q link.Queue, owner, repo string, number int, head, keep string) []link.QueuedAsk {
|
|
||||||
if number <= 0 || head == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []link.QueuedAsk
|
|
||||||
for _, a := range q.Asks {
|
|
||||||
c := a.Request.Check
|
|
||||||
switch {
|
|
||||||
case c == nil, link.IsTry(a.Request), c.Group != "", c.Number != number, a.ID == keep,
|
|
||||||
!strings.EqualFold(c.Owner, owner), !strings.EqualFold(c.Repo, repo),
|
|
||||||
a.State != link.AskWaiting && a.State != link.AskInFlight:
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ref := a.Request.Ref
|
|
||||||
if ref == "" {
|
|
||||||
ref = a.Ref
|
|
||||||
}
|
|
||||||
if ref == "" || ref == head || link.SameHead(ref, head) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, a)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// endSupersededChecks ends the checks a check of p at its head supersedes, keep being the new check's own ask, and
|
|
||||||
// says each in the log. A waiting one is cancelled — nothing recorded: a check builds nothing, and the forge's status
|
|
||||||
// of the old head is no longer the pull request's — and one a machine builds is killed there.
|
|
||||||
func endSupersededChecks(ctx context.Context, js *broker.JetStream, seat string, p link.PullUpdated, keep string) []string {
|
|
||||||
if p.Number <= 0 || p.Commit == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
current, err := forgePullHead(ctx, js, p.Owner, p.Repo, p.Number)
|
|
||||||
if err != nil {
|
|
||||||
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the forge could not say the pull request's "+
|
|
||||||
"current head, so checks of its other heads keep running: %v", p.Owner, p.Repo, p.Number, p.Commit, err)}
|
|
||||||
}
|
|
||||||
if current != p.Commit && !link.SameHead(current, p.Commit) {
|
|
||||||
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the pull request's head is %.12s now, not "+
|
|
||||||
"this one", p.Owner, p.Repo, p.Number, p.Commit, current)}
|
|
||||||
}
|
|
||||||
q, err := link.ReadQueue(ctx, js, seat)
|
|
||||||
if err != nil {
|
|
||||||
return []string{fmt.Sprintf("%s/%s#%d (%.8s): could not read the %s queue for checks of its earlier heads: %v",
|
|
||||||
p.Owner, p.Repo, p.Number, p.Commit, seat, err)}
|
|
||||||
}
|
|
||||||
var said []string
|
|
||||||
for _, a := range supersededChecks(q, p.Owner, p.Repo, p.Number, p.Commit, keep) {
|
|
||||||
old := a.Request.Ref
|
|
||||||
if old == "" {
|
|
||||||
old = a.Ref
|
|
||||||
}
|
|
||||||
what := fmt.Sprintf("check of %s/%s#%d at %.12s", p.Owner, p.Repo, p.Number, old)
|
|
||||||
var err error
|
|
||||||
if a.State == link.AskInFlight && a.On != "" {
|
|
||||||
var answer link.Answer
|
|
||||||
answer, err = link.AskSeatTool(ctx, js.Conn(), seat, "kill", a.On, map[string]string{"id": a.ID}, killAnswer)
|
|
||||||
if err == nil && answer.Error != "" {
|
|
||||||
err = fmt.Errorf("%s refused: %s", a.On, answer.Error)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
_, err = cancelAsk(ctx, js, nil, seat, a)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
said = append(said, fmt.Sprintf("not ended: %s (%s) — superseded by %.12s, but: %v", what, a.ID, p.Commit, err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
said = append(said, fmt.Sprintf("ended: %s — superseded by %.12s (%s, %s)", what, p.Commit, a.ID, a.State))
|
|
||||||
}
|
|
||||||
return said
|
|
||||||
}
|
|
||||||
|
|
||||||
// endingSupersededChecks ends them beside the announcement, so a kill waiting on its machine never holds the
|
|
||||||
// handler: on its own connection to the bus, within twice a kill's wait. A one-shot process (oneShot) ends them
|
|
||||||
// within the call instead, as its exit would cut them off.
|
|
||||||
func endingSupersededChecks(ctx context.Context, seat string, p link.PullUpdated, keep string, oneShot bool) {
|
|
||||||
end := func() {
|
|
||||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*killAnswer)
|
|
||||||
defer cancel()
|
|
||||||
js, err := dialTheBus()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "%s/%s#%d (%.8s): could not reach the bus to end checks of its earlier heads: %v\n",
|
|
||||||
p.Owner, p.Repo, p.Number, p.Commit, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
for _, line := range endSupersededChecks(ctx, js, seat, p, keep) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if oneShot {
|
|
||||||
end()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
go end()
|
|
||||||
}
|
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A new push to a pull request ends the check of its old head (novox/hq issue 726): the waiting one is dropped from
|
|
||||||
// the queue, the one in flight is killed on its machine, and each is said. A try, a walk's build, another pull
|
|
||||||
// request's check, a group's check and a recheck of the new head are left alone. On the bus alone: no store.
|
|
||||||
func TestANewHeadEndsTheChecksOfTheOldOnes(t *testing.T) {
|
|
||||||
js, _ := aBuildQueue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
const (
|
|
||||||
oldHead = "1111111111111111111111111111111111111111"
|
|
||||||
olderHead = "2222222222222222222222222222222222222222"
|
|
||||||
newHead = "3333333333333333333333333333333333333333"
|
|
||||||
)
|
|
||||||
publish := func(r link.BuildRequest) {
|
|
||||||
t.Helper()
|
|
||||||
body, _ := json.Marshal(r)
|
|
||||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
check := func(id string, number int, head string) link.BuildRequest {
|
|
||||||
return link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
|
||||||
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: number}}
|
|
||||||
}
|
|
||||||
// In flight first: taken by a holder, started on ace.
|
|
||||||
publish(check("check-running", 7, olderHead))
|
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
|
||||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
started, _ := json.Marshal(link.BuildStart{ID: "check-running", On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
|
||||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
publish(check("check-old", 7, oldHead))
|
|
||||||
publish(check("check-other-pull", 8, oldHead))
|
|
||||||
publish(check("check-recheck", 7, newHead))
|
|
||||||
publish(check("check-new", 7, newHead))
|
|
||||||
try := check("try-old", 7, oldHead)
|
|
||||||
try.Check.Number, try.Check.Try = 0, true
|
|
||||||
publish(try)
|
|
||||||
group := check("group-old", 7, oldHead)
|
|
||||||
group.Check.Group = "g1"
|
|
||||||
publish(group)
|
|
||||||
publish(link.BuildRequest{ID: "walk-build", Repository: "https://forge.example/novox/a.git", Ref: oldHead})
|
|
||||||
|
|
||||||
var mu sync.Mutex
|
|
||||||
var killed []string
|
|
||||||
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", map[string]link.ToolHandler{
|
|
||||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
|
||||||
var args struct{ ID string }
|
|
||||||
_ = json.Unmarshal(raw, &args)
|
|
||||||
mu.Lock()
|
|
||||||
killed = append(killed, args.ID)
|
|
||||||
mu.Unlock()
|
|
||||||
return map[string]any{"said": "killed " + args.ID}, nil
|
|
||||||
}}, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
forgeSays(t, newHead, nil)
|
|
||||||
said := endSupersededChecks(ctx, js, link.TheBuildMachine,
|
|
||||||
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
|
|
||||||
|
|
||||||
mu.Lock()
|
|
||||||
if len(killed) != 1 || killed[0] != "check-running" {
|
|
||||||
t.Errorf("the holder was asked to kill %v, want only the in-flight check of the older head", killed)
|
|
||||||
}
|
|
||||||
mu.Unlock()
|
|
||||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, found := q.Find("check-old"); found {
|
|
||||||
t.Error("the waiting check of the old head is still in the queue")
|
|
||||||
}
|
|
||||||
for _, id := range []string{"check-other-pull", "check-recheck", "check-new", "try-old", "group-old", "walk-build"} {
|
|
||||||
if _, found := q.Find(id); !found {
|
|
||||||
t.Errorf("%s was ended; only checks of the same pull request at another head end", id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
text := strings.Join(said, "\n")
|
|
||||||
for _, want := range []string{
|
|
||||||
"ended: check of novox/a#7 at " + oldHead[:12] + " — superseded by " + newHead[:12],
|
|
||||||
"ended: check of novox/a#7 at " + olderHead[:12] + " — superseded by " + newHead[:12],
|
|
||||||
} {
|
|
||||||
if !strings.Contains(text, want) {
|
|
||||||
t.Errorf("the log does not say %q:\n%s", want, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(said) != 2 {
|
|
||||||
t.Errorf("said %d lines, want 2:\n%s", len(said), text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Which asks a check supersedes, read from the queue alone.
|
|
||||||
func TestSupersededChecksAreOnlyThePullRequestsOtherHeads(t *testing.T) {
|
|
||||||
ask := func(id, state, ref string, c *link.CheckRequest) link.QueuedAsk {
|
|
||||||
return link.QueuedAsk{ID: id, State: state, Ref: ref, Request: link.BuildRequest{ID: id, Ref: ref, Check: c}}
|
|
||||||
}
|
|
||||||
pr := func(n int) *link.CheckRequest { return &link.CheckRequest{Owner: "Novox", Repo: "A", Number: n} }
|
|
||||||
head := "abcdefabcdefabcdef0123456789abcdef012345"
|
|
||||||
q := link.Queue{Asks: []link.QueuedAsk{
|
|
||||||
ask("old", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
|
||||||
ask("short-same", link.AskWaiting, head[:12], pr(7)),
|
|
||||||
ask("dead", link.AskDead, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
|
||||||
ask("other", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(8)),
|
|
||||||
ask("build", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", nil),
|
|
||||||
ask("keep", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
|
||||||
}}
|
|
||||||
got := supersededChecks(q, "novox", "a", 7, head, "keep")
|
|
||||||
if len(got) != 1 || got[0].ID != "old" {
|
|
||||||
t.Fatalf("superseded %+v, want only old", got)
|
|
||||||
}
|
|
||||||
if got := supersededChecks(q, "novox", "a", 0, head, ""); got != nil {
|
|
||||||
t.Errorf("a check of no pull request superseded %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeSays makes the forge name head as every pull request's current head, or fail with err.
|
|
||||||
func forgeSays(t *testing.T, head string, err error) {
|
|
||||||
t.Helper()
|
|
||||||
was := forgePullHead
|
|
||||||
forgePullHead = func(context.Context, *broker.JetStream, string, string, int) (string, error) { return head, err }
|
|
||||||
t.Cleanup(func() { forgePullHead = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// An older head heard after a newer one — an announcement out of order, one heard again after a restart, a recheck
|
|
||||||
// of a recorded head — ends nothing: the forge says the pull request's head is the newer one. Nor does a check whose
|
|
||||||
// pull request's head the forge cannot say.
|
|
||||||
func TestAnOlderHeadHeardLateEndsNothing(t *testing.T) {
|
|
||||||
js, _ := aBuildQueue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
const (
|
|
||||||
oldHead = "1111111111111111111111111111111111111111"
|
|
||||||
newHead = "3333333333333333333333333333333333333333"
|
|
||||||
)
|
|
||||||
for _, id := range []string{"check-new", "check-old-late"} {
|
|
||||||
head := newHead
|
|
||||||
if id == "check-old-late" {
|
|
||||||
head = oldHead
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
|
||||||
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: 7}})
|
|
||||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
late := link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: oldHead}
|
|
||||||
forgeSays(t, newHead, nil)
|
|
||||||
said := endSupersededChecks(ctx, js, link.TheBuildMachine, late, "check-old-late")
|
|
||||||
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
|
|
||||||
t.Errorf("an older head heard late said %q", said)
|
|
||||||
}
|
|
||||||
forgeSays(t, "", errors.New("nothing serves gitea.gitea_api"))
|
|
||||||
said = endSupersededChecks(ctx, js, link.TheBuildMachine,
|
|
||||||
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
|
|
||||||
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
|
|
||||||
t.Errorf("a head the forge could not confirm said %q", said)
|
|
||||||
}
|
|
||||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, id := range []string{"check-new", "check-old-late"} {
|
|
||||||
if _, found := q.Find(id); !found {
|
|
||||||
t.Errorf("%s was ended", id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -294,10 +294,10 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// notNow marks a store that could not be read right now, so the announcement is held rather than
|
// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so
|
||||||
// lost; anything else is returned as it was.
|
// the announcement is held rather than lost; anything else is returned as it was.
|
||||||
func notNow(err error) error {
|
func notNow(err error) error {
|
||||||
if inventory.Unreachable(err) {
|
if inventory.Unreachable(err) || errors.Is(err, inventory.ErrPlansBusy) {
|
||||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A plan moved by a seat verb is looked at as the verb ends (issue 974, review 1): the verb runs as a child process
|
||||||
|
// whose saves the serving controller does not hear, so `deliver`, which lets a walk go after the operator's approval,
|
||||||
|
// would leave its first tier waiting for an unrelated event.
|
||||||
|
func TestAVerbThatMayMoveAPlanHasTheServingControllerLookWhenItEnds(t *testing.T) {
|
||||||
|
var looks, failures atomic.Int32
|
||||||
|
l := lookerCounting(&failures, &looks, nil)
|
||||||
|
lookerMu.Lock()
|
||||||
|
theLooker = l
|
||||||
|
lookerMu.Unlock()
|
||||||
|
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go l.run(ctx)
|
||||||
|
|
||||||
|
was := runTheVerb
|
||||||
|
runTheVerb = func(context.Context, []string) (verbAnswer, error) { return verbAnswer{}, nil }
|
||||||
|
t.Cleanup(func() { runTheVerb = was })
|
||||||
|
handlers, _, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args string
|
||||||
|
look bool
|
||||||
|
}{
|
||||||
|
{"deliver", `{"plan":"p"}`, true},
|
||||||
|
{"plans", `{"stop":"plan-1","why":"w"}`, true},
|
||||||
|
{"plans", `{}`, false},
|
||||||
|
{"nodes", `{}`, false},
|
||||||
|
} {
|
||||||
|
before := looks.Load()
|
||||||
|
h := handlers[c.verb]
|
||||||
|
if h == nil {
|
||||||
|
t.Fatalf("no verb %s", c.verb)
|
||||||
|
}
|
||||||
|
if _, err := h(ctx, []byte(c.args)); err != nil {
|
||||||
|
t.Fatalf("%s %s: %v", c.verb, c.args, err)
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if got := looks.Load() > before; got != c.look {
|
||||||
|
t.Errorf("%s %s: look asked = %v; want %v", c.verb, c.args, got, c.look)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -70,10 +70,6 @@ type signalFacts struct {
|
|||||||
loop loopFacts
|
loop loopFacts
|
||||||
loopErr error
|
loopErr error
|
||||||
|
|
||||||
merges []missedMerge
|
|
||||||
mergesPassed time.Time
|
|
||||||
mergesErr error
|
|
||||||
|
|
||||||
asks []askFacts
|
asks []askFacts
|
||||||
asksErr error
|
asksErr error
|
||||||
|
|
||||||
@@ -358,10 +354,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
|||||||
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
||||||
}
|
}
|
||||||
f.loop, f.loopErr = w.gatherLoop()
|
f.loop, f.loopErr = w.gatherLoop()
|
||||||
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
|
||||||
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
|
||||||
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
|
||||||
}
|
|
||||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||||
if open, err := w.keeper.Open(ctx); err != nil {
|
if open, err := w.keeper.Open(ctx); err != nil {
|
||||||
f.standingsErr = err
|
f.standingsErr = err
|
||||||
@@ -767,9 +759,6 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
|||||||
// and the answer chosen is performed on its warrant.
|
// and the answer chosen is performed on its warrant.
|
||||||
startAsking(watching, open, server, bus.Conn, keeper)
|
startAsking(watching, open, server, bus.Conn, keeper)
|
||||||
go forgettingOldHeals(watching, open.inventory)
|
go forgettingOldHeals(watching, open.inventory)
|
||||||
// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
|
|
||||||
// deliveries asked after it even when nothing new is asked.
|
|
||||||
go keepingTheQueueInOrder(watching, open, server.JetStream())
|
|
||||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||||
broker.ConditionsBucket, conditions.Seat)
|
broker.ConditionsBucket, conditions.Seat)
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// The pull requests' checks have work queues of their own (novox/hq ADR 0406 §3, issue 974).
|
||||||
|
//
|
||||||
|
// A delivery build goes on the build seat's own queue, `mesh.seat.node-build-agent.accept.build`. A check of a pull
|
||||||
|
// request marked ready goes on a second work-queue stream and every other check on a third. A builder takes from the
|
||||||
|
// first, then the second, then the third; the others are taken in the order asked. **Each check stream has one subject
|
||||||
|
// per pull request and keeps only the newest request** (a limit of one message a subject, the older discarded), so a
|
||||||
|
// newer request replaces one still waiting and a pull request has at most one waiting check: the stream's rule, not a
|
||||||
|
// pass of the controller that moves asks about. Nothing pauses the worker or deletes and republishes a request.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ChecksReadyStream holds the checks of pull requests marked ready; ChecksOtherStream every other check.
|
||||||
|
ChecksReadyStream = "CHECKS_READY"
|
||||||
|
ChecksOtherStream = "CHECKS_OTHER"
|
||||||
|
// ChecksReadyWorker and ChecksOtherWorker are the durable consumers the build seat's holders share on them, as
|
||||||
|
// they share the seat's worker on its own queue (novox/hq ADR 0190).
|
||||||
|
ChecksReadyWorker = "CHECKS_READY_worker"
|
||||||
|
ChecksOtherWorker = "CHECKS_OTHER_worker"
|
||||||
|
// TheBuildSeat is the role whose checks these streams queue. The retired build role's checks stay on its own queue.
|
||||||
|
TheBuildSeat = "node-build-agent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PullsBucket is mesh-delivery's key per pull request: its head, the trunk commit its current check is asked against and
|
||||||
|
// its verdict (ADR 0406 §4). A running check watches its key.
|
||||||
|
var PullsBucket = BucketName("mesh-delivery", "pulls")
|
||||||
|
|
||||||
|
// CheckSubjectPrefix is where a seat's checks of one kind are asked: `mesh.seat.<seat>.check.ready.` or `.other.`. The
|
||||||
|
// `check` token keeps the subjects out of the seat's own queue, which takes `accept.>`.
|
||||||
|
func CheckSubjectPrefix(seat string, ready bool) string {
|
||||||
|
kind := "other"
|
||||||
|
if ready {
|
||||||
|
kind = "ready"
|
||||||
|
}
|
||||||
|
return "mesh.seat." + seat + ".check." + kind + "."
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckStreams are the two check queues, in the order a builder takes from them.
|
||||||
|
func CheckStreams() []Stream {
|
||||||
|
return []Stream{
|
||||||
|
{
|
||||||
|
Name: ChecksReadyStream,
|
||||||
|
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">"},
|
||||||
|
Retention: RetentionWorkQueue,
|
||||||
|
MaxAge: 7 * 24 * 60 * 60,
|
||||||
|
MaxMsgsPerSubject: 1,
|
||||||
|
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
|
||||||
|
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
|
||||||
|
"request still waiting",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: ChecksOtherStream,
|
||||||
|
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, false) + ">"},
|
||||||
|
Retention: RetentionWorkQueue,
|
||||||
|
MaxAge: 7 * 24 * 60 * 60,
|
||||||
|
MaxMsgsPerSubject: 1,
|
||||||
|
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
|
||||||
|
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckWorkers are the consumers the build seat's holders pull the check queues through: as the seat's own worker is,
|
||||||
|
// one for every holder, one ask at a time, acknowledged when the check is done.
|
||||||
|
func CheckWorkers() []Consumer {
|
||||||
|
worker := func(name, stream string, ready bool) Consumer {
|
||||||
|
return Consumer{
|
||||||
|
Name: name,
|
||||||
|
Stream: stream,
|
||||||
|
Filters: []string{CheckSubjectPrefix(TheBuildSeat, ready) + ">"},
|
||||||
|
AckWaitSeconds: 60,
|
||||||
|
MaxDeliver: 5,
|
||||||
|
Why: "every holder of " + TheBuildSeat + " pulls one check at a time from this worker and acknowledges after " +
|
||||||
|
"the check is done, so a crash mid-check redelivers rather than loses (novox/hq ADR 0406 §3)",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []Consumer{
|
||||||
|
worker(ChecksReadyWorker, ChecksReadyStream, true),
|
||||||
|
worker(ChecksOtherWorker, ChecksOtherStream, false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// holderCheckGrants are what a holder of the build seat is granted to take checks: ask about the two workers, pull from
|
||||||
|
// them and acknowledge, and nothing of the subjects themselves.
|
||||||
|
func holderCheckGrants() []string {
|
||||||
|
var pub []string
|
||||||
|
for _, w := range CheckWorkers() {
|
||||||
|
pub = append(pub,
|
||||||
|
"$JS.API.CONSUMER.INFO."+w.Stream+"."+w.Name,
|
||||||
|
"$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Name,
|
||||||
|
"$JS.ACK."+w.Stream+"."+w.Name+".>")
|
||||||
|
}
|
||||||
|
// And the bucket of the checks they are running: each writes the key of its own check, reads another's, and the bucket
|
||||||
|
// is a controller's own bucket the build seat's holders write by design (ADR 0406 §3).
|
||||||
|
pub = append(pub,
|
||||||
|
"$KV."+CheckRunningBucket+".>",
|
||||||
|
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
|
||||||
|
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
|
||||||
|
return pub
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsCheckStream says a stream is one of the check queues.
|
||||||
|
func IsCheckStream(stream string) bool {
|
||||||
|
return stream == ChecksReadyStream || stream == ChecksOtherStream
|
||||||
|
}
|
||||||
|
|
||||||
|
// isCheckAsk says a message of a check queue is an ask the controller itself makes.
|
||||||
|
func isCheckAsk(stream, subject string) bool {
|
||||||
|
switch stream {
|
||||||
|
case ChecksReadyStream:
|
||||||
|
return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, true))
|
||||||
|
case ChecksOtherStream:
|
||||||
|
return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, false))
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// The build agent's credential is the module principal `<node>.build-agent` (Username, KindModule): its manifest's
|
||||||
|
// `reads: ["mesh-delivery.pulls"]` reaches that principal as a read and a watch of mesh-delivery's bucket of pull requests'
|
||||||
|
// keys — the direct get of a key, the consumer a watch makes, the stream's info — and as no write and no direct
|
||||||
|
// subscription of the bucket's subjects (novox/hq ADR 0406 §4, issue 974).
|
||||||
|
func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
|
||||||
|
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||||
|
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat},
|
||||||
|
Reads: []string{"mesh-delivery.pulls"}, PasswordHash: "x"}
|
||||||
|
if got := p.Username(); got != "one.build-agent" {
|
||||||
|
t.Fatalf("the build agent's credential is %q", got)
|
||||||
|
}
|
||||||
|
perms, err := PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range []string{
|
||||||
|
"$JS.API.STREAM.INFO.KV_mesh-delivery_pulls",
|
||||||
|
"$JS.API.DIRECT.GET.KV_mesh-delivery_pulls.>",
|
||||||
|
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery_pulls.>",
|
||||||
|
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery_pulls.>",
|
||||||
|
// And the two check workers it takes from.
|
||||||
|
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_READY.CHECKS_READY_worker",
|
||||||
|
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_OTHER.CHECKS_OTHER_worker",
|
||||||
|
"$JS.ACK.CHECKS_OTHER.CHECKS_OTHER_worker.>",
|
||||||
|
// And the bucket of the checks running, which it writes and reads (ADR 0406 §3).
|
||||||
|
"$KV." + CheckRunningBucket + ".>",
|
||||||
|
"$JS.API.DIRECT.GET.KV_" + CheckRunningBucket + ".>",
|
||||||
|
} {
|
||||||
|
has(t, perms.Publish, s)
|
||||||
|
}
|
||||||
|
hasNot(t, perms.Publish, "$KV."+PullsBucket+".>")
|
||||||
|
hasNot(t, perms.Publish, "mesh.seat.node-build-agent.check.>")
|
||||||
|
for _, s := range perms.Subscribe {
|
||||||
|
if s == "$KV."+PullsBucket+".>" {
|
||||||
|
t.Fatalf("the build agent subscribes the bucket's subjects directly: %v", perms.Subscribe)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -160,3 +160,87 @@ func TestNatsOnlyMeshConfigsAccountPutsToItsBuckets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **The controller asks mesh-config's `config` in its own name** (issue 925): on a real server whose users are
|
||||||
|
// composed from records that hold mesh-config on novox serving `config`, the controller's request on
|
||||||
|
// `mesh.seat.mesh-config.call.config.controller` reaches mesh-config's account and its reply reaches the
|
||||||
|
// controller's inbox; the same request under another caller's name is refused by the server.
|
||||||
|
func TestNatsTheControllerAsksMeshConfigInItsOwnName(t *testing.T) {
|
||||||
|
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seat := Seat{Name: "mesh-config", Scope: "mesh", Serves: []string{"config"}}
|
||||||
|
users, err := Users(Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{
|
||||||
|
"novox": {{Module: RuntimeModule}, {Module: "mesh-config", RunsAs: "mesh-config", Holds: []Seat{seat}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := range users {
|
||||||
|
users[i].PasswordHash = string(hash)
|
||||||
|
}
|
||||||
|
accounts, err := ComposeAccounts(users)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||||
|
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\n"+accounts), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opts, err := server.ProcessConfigFile(conf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||||
|
}
|
||||||
|
opts.NoLog, opts.NoSigs = true, true
|
||||||
|
s, err := server.NewServer(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go s.Start()
|
||||||
|
if !s.ReadyForConnections(10 * time.Second) {
|
||||||
|
t.Fatal("the bus did not come up")
|
||||||
|
}
|
||||||
|
t.Cleanup(s.Shutdown)
|
||||||
|
connect := func(user string) (*nats.Conn, chan error) {
|
||||||
|
t.Helper()
|
||||||
|
refused := make(chan error, 8)
|
||||||
|
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||||
|
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s cannot connect: %v", user, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(nc.Close)
|
||||||
|
return nc, refused
|
||||||
|
}
|
||||||
|
owner, _ := connect("novox.mesh-config")
|
||||||
|
subject := CalledSubject("mesh.seat.mesh-config.tool.config", ControllerName)
|
||||||
|
if subject != "mesh.seat.mesh-config.call.config.controller" {
|
||||||
|
t.Fatalf("the subject is %s", subject)
|
||||||
|
}
|
||||||
|
if _, err := owner.Subscribe("mesh.seat.mesh-config.call.config.*", func(m *nats.Msg) {
|
||||||
|
_ = m.Respond([]byte(`{"ok":true}`))
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := owner.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
controller, refused := connect(ControllerName)
|
||||||
|
reply, err := controller.Request(subject, []byte(`{"action":"set"}`), 5*time.Second)
|
||||||
|
if err != nil || string(reply.Data) != `{"ok":true}` {
|
||||||
|
t.Fatalf("the controller's own-name request got %v (%v), want mesh-config's reply", reply, err)
|
||||||
|
}
|
||||||
|
if err := controller.Publish(CalledSubject("mesh.seat.mesh-config.tool.config", "anchor.node-tools"), nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = controller.Flush()
|
||||||
|
select {
|
||||||
|
case err := <-refused:
|
||||||
|
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||||
|
t.Errorf("the server said %v, want a permissions violation", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Error("the server did not refuse the controller a request in another's name")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -38,6 +38,24 @@ var (
|
|||||||
// each ask by its id, its options and the actions they stand for, how it ended and whether the
|
// each ask by its id, its options and the actions they stand for, how it ended and whether the
|
||||||
// controller acted on its warrant — so a restart neither asks twice nor acts twice.
|
// controller acted on its warrant — so a restart neither asks twice nor acts twice.
|
||||||
AskedBucket = BucketName(ControllerSeat, "asked")
|
AskedBucket = BucketName(ControllerSeat, "asked")
|
||||||
|
// MergeWindowBucket holds the clock of every open batch's merge window (novox/hq ADR 0407 decision 1): per
|
||||||
|
// batch a key `<batch>.quiet`, made again by each merge into it and living the window, and a key
|
||||||
|
// `<batch>.most`, made once when it opens and living the window's most. The first to expire closes the
|
||||||
|
// batch, and the controller's watcher hears the delete marker. Written by the lease holder alone.
|
||||||
|
MergeWindowBucket = BucketName(ControllerSeat, "merge-window")
|
||||||
|
// RolloutSilentBucket holds the wait of every machine a rollout has sent a tier to (novox/hq ADR 0406 §5, issue
|
||||||
|
// 974): per send a key `<plan>.<tier>.<node>` that lives ApplySilentAfter, made when the controller sends that
|
||||||
|
// machine and deleted by the machine's report on that send. Only a missing report lets it expire, and the
|
||||||
|
// controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease
|
||||||
|
// holder alone.
|
||||||
|
RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent")
|
||||||
|
// CheckRunningBucket holds the pull request checks the builders are running now (novox/hq ADR 0406 §3, issue 974): per
|
||||||
|
// pull request a key `<owner>.<repo>.<number>` (or `group.<name>`) holding the head and trunk commit being checked, the
|
||||||
|
// builder and its build. A builder puts it when it starts a check and again on each heartbeat, and deletes it when the
|
||||||
|
// check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a
|
||||||
|
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
|
||||||
|
// design** — each the key of the check it runs.
|
||||||
|
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
|
||||||
)
|
)
|
||||||
|
|
||||||
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
|
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
|
||||||
@@ -79,6 +97,22 @@ const (
|
|||||||
MirrorMarkerTTL = 24 * time.Hour
|
MirrorMarkerTTL = 24 * time.Hour
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MergeWindowMarkerTTL is how long the marker an expired window key leaves is kept: long enough for a watcher
|
||||||
|
// that is up to hear it, short enough to leave nothing behind.
|
||||||
|
const MergeWindowMarkerTTL = time.Minute
|
||||||
|
|
||||||
|
// RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason.
|
||||||
|
const RolloutSilentMarkerTTL = time.Minute
|
||||||
|
|
||||||
|
// RolloutSilentKeptFor is the age of the silent keys' bucket: every report deletes a key and the delete leaves a marker, which
|
||||||
|
// with one value per key would stay for ever and fill the bucket. A key lives a quarter of an hour at most, so an hour
|
||||||
|
// takes nothing live.
|
||||||
|
const RolloutSilentKeptFor = time.Hour
|
||||||
|
|
||||||
|
// CheckRunningKeptFor is the age of a running check's key: a few heartbeats (ten seconds each) of a builder, so a builder that
|
||||||
|
// died is forgotten within two minutes and the check it ran is taken again then.
|
||||||
|
const CheckRunningKeptFor = 2 * time.Minute
|
||||||
|
|
||||||
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
|
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
|
||||||
const AskedKeptFor = 30 * 24 * time.Hour
|
const AskedKeptFor = 30 * 24 * time.Hour
|
||||||
|
|
||||||
@@ -106,13 +140,14 @@ const (
|
|||||||
func IsControllerBucket(bucket string) bool {
|
func IsControllerBucket(bucket string) bool {
|
||||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket
|
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||||
|
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||||
func ControllerBuckets() []string {
|
func ControllerBuckets() []string {
|
||||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket}
|
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||||
@@ -209,6 +244,52 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err)
|
return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err)
|
||||||
}
|
}
|
||||||
|
// **The merge window: keys that expire.** A key's own time to live is the window; the marker an expired key
|
||||||
|
// leaves is what a watcher hears, kept a minute (ADR 0407 decision 1).
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: MergeWindowBucket,
|
||||||
|
Description: "the clock of every open batch's merge window (novox/hq ADR 0407): `<batch>.quiet` lives the " +
|
||||||
|
"window and is made again by each merge, `<batch>.most` lives the most; the first to expire closes the " +
|
||||||
|
"batch; written by the controller alone",
|
||||||
|
History: 1,
|
||||||
|
MaxValueSize: 4 << 10,
|
||||||
|
MaxBytes: 8 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
LimitMarkerTTL: MergeWindowMarkerTTL,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", MergeWindowBucket, err)
|
||||||
|
}
|
||||||
|
// **The rollout's silent keys: keys that expire** (ADR 0406 §5, issue 974). A key lives as long as the machine
|
||||||
|
// may stay silent; the machine's report deletes it, and a marker of expiry is what the watcher hears.
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: RolloutSilentBucket,
|
||||||
|
Description: "the wait of every machine a rollout sent a tier to (novox/hq ADR 0406): `<plan>.<tier>.<node>` " +
|
||||||
|
"lives the time a machine may stay silent and is deleted by its report; one that expires names the " +
|
||||||
|
"machine silent; written by the controller alone",
|
||||||
|
History: 1,
|
||||||
|
TTL: RolloutSilentKeptFor,
|
||||||
|
MaxValueSize: 4 << 10,
|
||||||
|
MaxBytes: 8 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
LimitMarkerTTL: RolloutSilentMarkerTTL,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err)
|
||||||
|
}
|
||||||
|
// **The checks running: a key that ages** (ADR 0406 §3, issue 974). One message a key, each renewal a new one, so the
|
||||||
|
// age of the bucket is how long a key outlives the builder that stopped renewing it.
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: CheckRunningBucket,
|
||||||
|
Description: "the pull request checks the builders are running now (novox/hq ADR 0406): `<pull request>` holds the " +
|
||||||
|
"head and trunk commit, the builder and its build; put by the builder when it starts and at each heartbeat, " +
|
||||||
|
"deleted when the check ends, gone within two minutes of a builder that died; written by the build seat's holders",
|
||||||
|
History: 1,
|
||||||
|
TTL: CheckRunningKeptFor,
|
||||||
|
MaxValueSize: 4 << 10,
|
||||||
|
MaxBytes: 8 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
|
||||||
|
}
|
||||||
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
||||||
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
||||||
for _, m := range []struct{ bucket, why string }{
|
for _, m := range []struct{ bucket, why string }{
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package broker
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -226,3 +227,278 @@ func TestTheControllerMayAskTheVaultInItsOwnName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **The controller may ask mesh-config's `config` in its own name** (issue 925): move-config and the settings verb
|
||||||
|
// set a trusted value through mesh-config, which takes it as the controller carries it only from the subject that
|
||||||
|
// names the controller as caller. Granted when the records hold a holder serving the verb, and not otherwise;
|
||||||
|
// the installer's first user list carries none.
|
||||||
|
func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) {
|
||||||
|
config := Declared{Module: "mesh-config", Holds: []Seat{{Name: "mesh-config", Serves: []string{"config"}}}}
|
||||||
|
users, err := Users(Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {config}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(users[0].AsksTheConfig, ",") != "config" {
|
||||||
|
t.Fatalf("composed %v", users[0].AsksTheConfig)
|
||||||
|
}
|
||||||
|
p, err := PermissionsFor(users[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(p.Publish, "mesh.seat.mesh-config.call.config.controller") {
|
||||||
|
t.Errorf("the controller may not publish mesh.seat.mesh-config.call.config.controller: %v", p.Publish)
|
||||||
|
}
|
||||||
|
if !MayPublish(p, CalledSubject("mesh.seat.mesh-config.tool.config", ControllerName)) {
|
||||||
|
t.Error("the controller may not ask mesh-config's config in its own name")
|
||||||
|
}
|
||||||
|
if MayPublish(p, CalledSubject("mesh.seat.mesh-config.tool.config", "anchor.node-tools")) {
|
||||||
|
t.Error("the controller may ask mesh-config's config in another's name")
|
||||||
|
}
|
||||||
|
// No holder serving it composes no grant, nor does genesis.
|
||||||
|
other := Declared{Module: "mesh-config", Holds: []Seat{{Name: "mesh-config", Serves: []string{"list"}}}}
|
||||||
|
for _, r := range []Records{{Nodes: []string{"novox"}}, {Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {other}}}} {
|
||||||
|
users, _ = Users(r)
|
||||||
|
if len(users[0].AsksTheConfig) != 0 {
|
||||||
|
t.Errorf("composed %v without a holder serving config", users[0].AsksTheConfig)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
genesis, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
for _, s := range genesis.Publish {
|
||||||
|
if strings.Contains(s, "mesh-config") {
|
||||||
|
t.Errorf("the installer's controller may publish %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The merge window is a bucket whose keys expire** (novox/hq ADR 0407 decision 1, issue 973): a key made with a
|
||||||
|
// time to live goes when it is up, and a watcher on the bucket hears it go. The controller may write it, and it is
|
||||||
|
// no undeclared state.
|
||||||
|
func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) {
|
||||||
|
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") {
|
||||||
|
t.Error("the controller may not write the merge window's bucket")
|
||||||
|
}
|
||||||
|
if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) {
|
||||||
|
t.Error("the merge window's bucket is not asserted as the controller's own")
|
||||||
|
}
|
||||||
|
js, err := Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, MergeWindowBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer w.Stop()
|
||||||
|
if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case e := <-w.Updates():
|
||||||
|
if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("no watcher heard the key expire")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a
|
||||||
|
// time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The
|
||||||
|
// controller may write it, and it is no undeclared state.
|
||||||
|
func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) {
|
||||||
|
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") {
|
||||||
|
t.Error("the controller may not write the rollout's silent keys' bucket")
|
||||||
|
}
|
||||||
|
if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) {
|
||||||
|
t.Error("the rollout's silent keys' bucket is not asserted as the controller's own")
|
||||||
|
}
|
||||||
|
js, err := Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer w.Stop()
|
||||||
|
if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case e := <-w.Updates():
|
||||||
|
if e == nil || e.Operation() == jetstream.KeyValuePut {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete {
|
||||||
|
t.Fatalf("a report's delete was heard as %v", e.Operation())
|
||||||
|
}
|
||||||
|
if e.Key() == "plan-1.0.g14" {
|
||||||
|
if e.Operation() != jetstream.KeyValuePurge {
|
||||||
|
t.Fatalf("an expiry was heard as %v, not a purge", e.Operation())
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("no watcher heard the key expire")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The silent keys' bucket ages**, or its delete markers would fill it (issue 974, review 1): every report deletes a key,
|
||||||
|
// the delete leaves a marker, and with one value per key and no age the markers stay for ever. A bucket made before the
|
||||||
|
// age was set is brought to it by the next start, which the server accepts as an update of an existing bucket.
|
||||||
|
func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) {
|
||||||
|
js, err := Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// As the first build made it: no age.
|
||||||
|
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1,
|
||||||
|
MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatalf("a bucket made without the age is not brought to it: %v", err)
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
status, err := kv.Status(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if status.TTL() != RolloutSilentKeptFor {
|
||||||
|
t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq
|
||||||
|
// ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge
|
||||||
|
// waiting in the merges consumer is not that consumer's, and nothing would catch it up.
|
||||||
|
func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) {
|
||||||
|
js, err := Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
if err := AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := AssertMeshConsumers(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range MeshConsumers() {
|
||||||
|
if c.Stream == EventsStream && c.Name == MergesConsumerName {
|
||||||
|
if c.Resettable != "" || c.MaxDeliver != 0 || len(c.Filters) != 1 || c.Filters[0] != ControllerFollows[3] {
|
||||||
|
t.Errorf("the merges consumer is %+v; want one filter, no max-deliver, not resettable", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.Stream == EventsStream && c.Name == ControllerName {
|
||||||
|
for _, f := range c.Filters {
|
||||||
|
if f == ControllerFollows[3] {
|
||||||
|
t.Errorf("the events consumer still follows the merges, which the merges consumer takes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
|
||||||
|
if err != nil || merges.NumPending != 1 {
|
||||||
|
t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err)
|
||||||
|
}
|
||||||
|
if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
|
||||||
|
if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 {
|
||||||
|
t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the
|
||||||
|
// `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has
|
||||||
|
// made it, because it starts where `controller` has acknowledged up to, not from now.
|
||||||
|
func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T) {
|
||||||
|
js, err := Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
if err := AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := &nats.ConsumerConfig{Durable: ControllerName, AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second,
|
||||||
|
MaxDeliver: 5, MaxAckPending: 1, DeliverSubject: "_DELIVER.controller.EVENTS", DeliverPolicy: nats.DeliverNewPolicy,
|
||||||
|
FilterSubjects: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName))}
|
||||||
|
if _, err := js.Context().AddConsumer(EventsStream, old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := AssertMeshConsumers(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
|
||||||
|
if err != nil || merges.NumPending != 1 {
|
||||||
|
t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -36,6 +36,12 @@ type Consumer struct {
|
|||||||
// them over, and a loop that acts on one message at a time is the property the store window
|
// them over, and a loop that acts on one message at a time is the property the store window
|
||||||
// depends on. A queue group implies this, because a group has nothing to pull from.
|
// depends on. A queue group implies this, because a group has nothing to pull from.
|
||||||
Push bool
|
Push bool
|
||||||
|
// TakesOverFrom names a consumer of the same stream whose filter this one takes over (issue 973): made while
|
||||||
|
// that consumer still filters this one's subject, it starts after what that consumer has acknowledged, so a
|
||||||
|
// message the old one held unacknowledged is not lost to the change. Otherwise it starts as FromNow says.
|
||||||
|
TakesOverFrom string
|
||||||
|
// DeliverSubject, when set, is where a push consumer delivers instead of DeliverSubjectFor's.
|
||||||
|
DeliverSubject string
|
||||||
// AckWaitSeconds before an unacknowledged delivery is redelivered.
|
// AckWaitSeconds before an unacknowledged delivery is redelivered.
|
||||||
AckWaitSeconds int
|
AckWaitSeconds int
|
||||||
// MaxDeliver is how often a message is handed over before the consumer gives it up; zero for no
|
// MaxDeliver is how often a message is handed over before the consumer gives it up; zero for no
|
||||||
|
|||||||
@@ -343,6 +343,9 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
|||||||
if c.FromNow {
|
if c.FromNow {
|
||||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||||
}
|
}
|
||||||
|
if from := j.ackFloorOfTheConsumerItTakesOver(c); from > 0 {
|
||||||
|
want.DeliverPolicy, want.OptStartSeq = nats.DeliverByStartSequencePolicy, from
|
||||||
|
}
|
||||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||||
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||||
}
|
}
|
||||||
@@ -352,6 +355,26 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ackFloorOfTheConsumerItTakesOver is the stream sequence a consumer made now starts at so as to lose nothing the
|
||||||
|
// consumer it takes over from (Consumer.TakesOverFrom) has not acknowledged: that consumer's ack floor plus one, when
|
||||||
|
// it exists and still filters this one's subject; zero otherwise.
|
||||||
|
func (j *JetStream) ackFloorOfTheConsumerItTakesOver(c Consumer) uint64 {
|
||||||
|
if c.TakesOverFrom == "" || len(c.Filters) != 1 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
have, err := j.js.ConsumerInfo(c.Stream, c.TakesOverFrom)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
filters := append([]string{have.Config.FilterSubject}, have.Config.FilterSubjects...)
|
||||||
|
for _, f := range filters {
|
||||||
|
if f == c.Filters[0] {
|
||||||
|
return have.AckFloor.Stream + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
// ConsumerState is what a reset says about a consumer, before and after.
|
// ConsumerState is what a reset says about a consumer, before and after.
|
||||||
type ConsumerState struct {
|
type ConsumerState struct {
|
||||||
DeliverPolicy string
|
DeliverPolicy string
|
||||||
|
|||||||
+63
-1
@@ -141,6 +141,12 @@ type Principal struct {
|
|||||||
// controller asks. Composed from the records like AsksIssued, so the installer's first user list carries none.
|
// controller asks. Composed from the records like AsksIssued, so the installer's first user list carries none.
|
||||||
AsksTheVault []string
|
AsksTheVault []string
|
||||||
|
|
||||||
|
// AsksTheConfig are the mesh-config verbs the controller asks in its own name (VerbsTheControllerAsksTheConfig,
|
||||||
|
// issue 925), those mesh-config's holder serves: on the subject that names its caller, so mesh-config takes a
|
||||||
|
// value as the controller carries it (ADR 0348 §8). Composed from the records like AsksTheVault, so the
|
||||||
|
// installer's first user list carries none.
|
||||||
|
AsksTheConfig []string
|
||||||
|
|
||||||
// AsksTheDesk says this principal is the vault's holder, which asks the desks' secret prompt in its own name
|
// AsksTheDesk says this principal is the vault's holder, which asks the desks' secret prompt in its own name
|
||||||
// (novox/hq ADR 0349 clause 9, issue 723): `node-launcher.secret` on any machine, on the subject that names
|
// (novox/hq ADR 0349 clause 9, issue 723): `node-launcher.secret` on any machine, on the subject that names
|
||||||
// its caller, and nothing else of that seat. With AsksIssued on the same principal, the vault's whole reach
|
// its caller, and nothing else of that seat. With AsksIssued on the same principal, the vault's whole reach
|
||||||
@@ -211,6 +217,9 @@ var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
|||||||
// original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats'
|
// original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats'
|
||||||
// queues).
|
// queues).
|
||||||
func TheControllersAsk(stream, subject string) bool {
|
func TheControllersAsk(stream, subject string) bool {
|
||||||
|
if isCheckAsk(stream, subject) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
for _, seat := range seatsTheControllerAsks {
|
for _, seat := range seatsTheControllerAsks {
|
||||||
if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") {
|
if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") {
|
||||||
return true
|
return true
|
||||||
@@ -316,6 +325,12 @@ var VerbsTheControllerAsksTheVault = []SeatVerb{{Seat: "mesh-vault", Verb: "acce
|
|||||||
{Seat: "mesh-vault", Verb: "secret-ask"}, {Seat: "mesh-vault", Verb: "issued"}, {Seat: "mesh-vault", Verb: "rotate"},
|
{Seat: "mesh-vault", Verb: "secret-ask"}, {Seat: "mesh-vault", Verb: "issued"}, {Seat: "mesh-vault", Verb: "rotate"},
|
||||||
{Seat: "mesh-vault", Verb: "move-asks"}, {Seat: "mesh-vault", Verb: "get"}}
|
{Seat: "mesh-vault", Verb: "move-asks"}, {Seat: "mesh-vault", Verb: "get"}}
|
||||||
|
|
||||||
|
// VerbsTheControllerAsksTheConfig are the mesh-config seat's verbs the controller calls in its own name (issue 925):
|
||||||
|
// `config`, the one verb mesh-config serves, which move-config uses to set each value and the settings verb to set,
|
||||||
|
// clear and propose through mesh-config. mesh-config takes a value as the controller carries it only from the
|
||||||
|
// subject that names the controller as caller (ADR 0348 §8); granted as Principal.AsksTheConfig says.
|
||||||
|
var VerbsTheControllerAsksTheConfig = []SeatVerb{{Seat: "mesh-config", Verb: "config"}}
|
||||||
|
|
||||||
// VaultMoveIn is the vault's verb a machine's node-engine asks in its own name to hand in a value sealed to its
|
// VaultMoveIn is the vault's verb a machine's node-engine asks in its own name to hand in a value sealed to its
|
||||||
// machine (design 57's V3, issue 676); granted as Principal.MovesIntoTheVault says.
|
// machine (design 57's V3, issue 676); granted as Principal.MovesIntoTheVault says.
|
||||||
var VaultMoveIn = SeatVerb{Seat: "mesh-vault", Verb: "move-in"}
|
var VaultMoveIn = SeatVerb{Seat: "mesh-vault", Verb: "move-in"}
|
||||||
@@ -356,6 +371,37 @@ var DegradableGrants = []DegradableGrant{
|
|||||||
Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb,
|
Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb,
|
||||||
Reason: "refused: the controller raises the secret-given condition as before (ADR 0380 §9)",
|
Reason: "refused: the controller raises the secret-given condition as before (ADR 0380 §9)",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Subject: "$JS.ACK.EVENTS." + MergesConsumerName + ".>",
|
||||||
|
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at " +
|
||||||
|
"its first push, which is when it acknowledges a merge (ADR 0407 decision 3, issue 973)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Subject: "$KV." + MergeWindowBucket + ".>",
|
||||||
|
Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " +
|
||||||
|
"window setting says (ADR 0407 decision 1, issue 973)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Subject: "mesh.seat." + TheBuildSeat + ".check.>",
|
||||||
|
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
|
||||||
|
"first push, and a check asked before that is asked again by the next announcement of its head (ADR 0406 §3, " +
|
||||||
|
"issue 974)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Subject: "$KV." + CheckRunningBucket + ".>",
|
||||||
|
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
|
||||||
|
"holders do (ADR 0406 §3, issue 974)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
|
||||||
|
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
|
||||||
|
"be said is printed and goes on (ADR 0406 §5, issue 974)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Subject: "$KV." + RolloutSilentBucket + ".>",
|
||||||
|
Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " +
|
||||||
|
"for that wait (ADR 0406 §5, issue 974)",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||||
@@ -543,6 +589,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// the role, and whichever machine holding it is idle takes it.
|
// the role, and whichever machine holding it is idle takes it.
|
||||||
for _, seat := range seatsTheControllerAsks {
|
for _, seat := range seatsTheControllerAsks {
|
||||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||||
|
// **And its checks, on the two queues of their own** (novox/hq ADR 0406 §3, issue 974): one subject a pull
|
||||||
|
// request on each, under `check`; the retired role's checks stay on its accepts.
|
||||||
|
if seat == TheBuildSeat {
|
||||||
|
pub = append(pub, "mesh.seat."+seat+".check.>")
|
||||||
|
}
|
||||||
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
|
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
|
||||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||||
@@ -623,7 +674,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||||
// ever, refused by the list it already has.
|
// ever, refused by the list it already has.
|
||||||
sub = append(sub, ControllerFollows...)
|
sub = append(sub, ControllerFollows...)
|
||||||
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>")
|
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>", "$JS.ACK.EVENTS."+MergesConsumerName+".>")
|
||||||
|
|
||||||
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
|
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
|
||||||
// permits one reply to the reply subject of a message the user received — and a message a
|
// permits one reply to the reply subject of a message the user received — and a message a
|
||||||
@@ -851,6 +902,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
||||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
||||||
"$JS.ACK."+stream+"."+worker+".>")
|
"$JS.ACK."+stream+"."+worker+".>")
|
||||||
|
// And the checks' two workers, for the build role they are queued for (novox/hq ADR 0406 §3).
|
||||||
|
if s.Name == TheBuildSeat {
|
||||||
|
pub = append(pub, holderCheckGrants()...)
|
||||||
|
}
|
||||||
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
|
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
|
||||||
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
|
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
|
||||||
// moment the controller cancelled it ends it instead of building it. Read, never written:
|
// moment the controller cancelled it ends it instead of building it. Read, never written:
|
||||||
@@ -1089,6 +1144,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
pub = append(pub, "mesh.seat."+v.Seat+"."+CallKind+"."+v.Verb+"."+CallerToken(p.Username()))
|
pub = append(pub, "mesh.seat."+v.Seat+"."+CallKind+"."+v.Verb+"."+CallerToken(p.Username()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **And mesh-config's verb** (issue 925): it takes a trusted value from the controller alone, known from the
|
||||||
|
// subject's last token.
|
||||||
|
for _, v := range VerbsTheControllerAsksTheConfig {
|
||||||
|
if slices.Contains(p.AsksTheConfig, v.Verb) {
|
||||||
|
pub = append(pub, "mesh.seat."+v.Seat+"."+CallKind+"."+v.Verb+"."+CallerToken(p.Username()))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(pub)
|
sort.Strings(pub)
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
|||||||
states = append(states, link.KeyChecked)
|
states = append(states, link.KeyChecked)
|
||||||
// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||||
states = append(states, link.KeyPlanMoved)
|
states = append(states, link.KeyPlanMoved)
|
||||||
|
// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||||
|
states = append(states, link.KeySent)
|
||||||
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||||
states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged)
|
states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged)
|
||||||
for _, event := range states {
|
for _, event := range states {
|
||||||
|
|||||||
+53
-11
@@ -147,7 +147,7 @@ func OriginalOfAgain(subject string) (string, bool) {
|
|||||||
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
|
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
|
||||||
// tools on core NATS, where a lost call is a timeout the caller already handles).
|
// tools on core NATS, where a lost call is a timeout the caller already handles).
|
||||||
func MeshStreams() []Stream {
|
func MeshStreams() []Stream {
|
||||||
return []Stream{
|
return append([]Stream{
|
||||||
{
|
{
|
||||||
Name: "CONTROL",
|
Name: "CONTROL",
|
||||||
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
|
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
|
||||||
@@ -221,7 +221,7 @@ func MeshStreams() []Stream {
|
|||||||
"failed with the reason (novox/hq ADR 0349 §4): what a walk's end and a gate wait for (ADR 0362, " +
|
"failed with the reason (novox/hq ADR 0349 §4): what a walk's end and a gate wait for (ADR 0362, " +
|
||||||
"ADR 0349 §15), read by subject; a week, as the events are",
|
"ADR 0349 §15), read by subject; a week, as the events are",
|
||||||
},
|
},
|
||||||
}
|
}, CheckStreams()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProvAppliedStream keeps what each machine's provisioner client last said of each module's files, newest per
|
// ProvAppliedStream keeps what each machine's provisioner client last said of each module's files, newest per
|
||||||
@@ -233,6 +233,18 @@ const ProvAppliedStream = "PROV_APPLIED"
|
|||||||
// all (ProvisionerAppliedSubject).
|
// all (ProvisionerAppliedSubject).
|
||||||
const ProvAppliedSubjects = "prov.applied.>"
|
const ProvAppliedSubjects = "prov.applied.>"
|
||||||
|
|
||||||
|
// eventsTheControllerFollows are the events the controller's events consumer takes: all it follows but the forge's
|
||||||
|
// merges, which the merges consumer takes (MergesConsumerName).
|
||||||
|
func eventsTheControllerFollows() []string {
|
||||||
|
var out []string
|
||||||
|
for _, subject := range ControllerFollows {
|
||||||
|
if subject != ControllerFollows[3] {
|
||||||
|
out = append(out, subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// DeliverSubjectFor is where a push consumer's messages land.
|
// DeliverSubjectFor is where a push consumer's messages land.
|
||||||
//
|
//
|
||||||
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
|
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
|
||||||
@@ -248,6 +260,9 @@ const ProvAppliedSubjects = "prov.applied.>"
|
|||||||
// durable's name to its stream, and this subject was the one place that scoping was dropped. It
|
// durable's name to its stream, and this subject was the one place that scoping was dropped. It
|
||||||
// stays inside what a controller may already subscribe (`_DELIVER.controller.>`).
|
// stays inside what a controller may already subscribe (`_DELIVER.controller.>`).
|
||||||
func DeliverSubjectFor(c Consumer) string {
|
func DeliverSubjectFor(c Consumer) string {
|
||||||
|
if c.DeliverSubject != "" {
|
||||||
|
return c.DeliverSubject
|
||||||
|
}
|
||||||
return "_DELIVER." + c.Name + "." + c.Stream
|
return "_DELIVER." + c.Name + "." + c.Stream
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -317,6 +332,13 @@ func Overlaps() []string {
|
|||||||
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
|
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
|
||||||
const ControllerName = "controller"
|
const ControllerName = "controller"
|
||||||
|
|
||||||
|
// MergesConsumerName is the controller's durable consumer of the forge's merged event on EVENTS, apart from the
|
||||||
|
// events consumer (novox/hq ADR 0407 decision 3, issue 973). A merge is never dropped, so this consumer has no
|
||||||
|
// max-deliver, which would send a merge held five times to DEAD_LETTERS to wait for a person, and no healer
|
||||||
|
// resets it, which would drop the merges it holds. A consumer of its own because both of those are properties of
|
||||||
|
// a consumer, not of a subject.
|
||||||
|
const MergesConsumerName = "controller-merges"
|
||||||
|
|
||||||
// ControllerSeat is the role the control plane holds, and ControllerStates are the facts it states
|
// ControllerSeat is the role the control plane holds, and ControllerStates are the facts it states
|
||||||
// under it (novox/hq ADR 0134).
|
// under it (novox/hq ADR 0134).
|
||||||
//
|
//
|
||||||
@@ -346,6 +368,8 @@ var ControllerStates = []string{"applied", "refused", "built-before",
|
|||||||
"checked",
|
"checked",
|
||||||
// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
|
// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
|
||||||
"plan-moved",
|
"plan-moved",
|
||||||
|
// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||||
|
"sent",
|
||||||
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
|
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
|
||||||
// each registered module's declared config (design 57 F3).
|
// each registered module's declared config (design 57 F3).
|
||||||
"assignment-changed", "declared-config-changed", "predefined-changed"}
|
"assignment-changed", "declared-config-changed", "predefined-changed"}
|
||||||
@@ -441,7 +465,7 @@ const EventsStream = "EVENTS"
|
|||||||
// held through a store restart — the exact message the stream exists to protect — some minutes
|
// held through a store restart — the exact message the stream exists to protect — some minutes
|
||||||
// before the controller had finished deciding about it.
|
// before the controller had finished deciding about it.
|
||||||
func MeshConsumers() []Consumer {
|
func MeshConsumers() []Consumer {
|
||||||
return []Consumer{
|
return append([]Consumer{
|
||||||
{
|
{
|
||||||
Name: ControllerName,
|
Name: ControllerName,
|
||||||
Stream: "CONTROL",
|
Stream: "CONTROL",
|
||||||
@@ -450,10 +474,27 @@ func MeshConsumers() []Consumer {
|
|||||||
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
|
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
|
||||||
"max-deliver, because the store window's bound is the controller's own",
|
"max-deliver, because the store window's bound is the controller's own",
|
||||||
},
|
},
|
||||||
|
// The forge's merges (novox/hq ADR 0407 decision 3, issue 973): never dropped, so no max-deliver and no
|
||||||
|
// reset; a merge the controller cannot record yet is held with a growing delay until it can. **Asserted before
|
||||||
|
// the events consumer**, which stops filtering merges: the order of this list is the order of assertion.
|
||||||
|
{
|
||||||
|
Name: MergesConsumerName,
|
||||||
|
Stream: "EVENTS",
|
||||||
|
// Within what the controller may subscribe already (`_DELIVER.controller.>`).
|
||||||
|
DeliverSubject: "_DELIVER." + ControllerName + ".merges",
|
||||||
|
Filters: []string{ControllerFollows[3]},
|
||||||
|
Push: true,
|
||||||
|
AckWaitSeconds: 30,
|
||||||
|
MaxAckPending: 1,
|
||||||
|
FromNow: true,
|
||||||
|
TakesOverFrom: ControllerName,
|
||||||
|
Why: "the forge's merges, one at a time; no max-deliver and never reset, because a merge is never " +
|
||||||
|
"dropped and no person releases one",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
Name: ControllerName,
|
Name: ControllerName,
|
||||||
Stream: "EVENTS",
|
Stream: "EVENTS",
|
||||||
Filters: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName)),
|
Filters: append(eventsTheControllerFollows(), AgainFilter(ControllerName)),
|
||||||
Push: true,
|
Push: true,
|
||||||
AckWaitSeconds: 30,
|
AckWaitSeconds: 30,
|
||||||
MaxDeliver: 5,
|
MaxDeliver: 5,
|
||||||
@@ -463,12 +504,13 @@ func MeshConsumers() []Consumer {
|
|||||||
MaxAckPending: 1,
|
MaxAckPending: 1,
|
||||||
FromNow: true,
|
FromNow: true,
|
||||||
// **The one consumer the mesh resets by itself** (healer H4, issue 248): it fell a week
|
// **The one consumer the mesh resets by itself** (healer H4, issue 248): it fell a week
|
||||||
// behind once and held every merge after it. What a reset drops is caught up elsewhere —
|
// behind once and held every merge after it. What a reset drops is caught up elsewhere — a
|
||||||
// a merge by the catch-up pass that reads the forge (issue 266), a build's outcome from the
|
// build's outcome from the build records a plan settles from (issue 214), a provider's failing
|
||||||
// build records a plan settles from (issue 214), a provider's failing word by the provider
|
// word by the provider saying it again every quarter of an hour (ADR 0224). **Not the merges**:
|
||||||
// saying it again every quarter of an hour (ADR 0224).
|
// they are the merges consumer's below, which a reset leaves alone (ADR 0407 decision 3).
|
||||||
Resettable: "what it drops is caught up: merges by the catch-up pass (issue 266), build outcomes " +
|
Resettable: "what it drops is caught up: build outcomes from the build records (issue 214), a " +
|
||||||
"from the build records (issue 214), a provider's failing word said again (ADR 0224)",
|
"provider's failing word said again (ADR 0224); the forge's merges are not its, they have a " +
|
||||||
|
"consumer of their own that no reset touches",
|
||||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||||
"max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " +
|
"max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " +
|
||||||
"a person delivers it again or drops it",
|
"a person delivers it again or drops it",
|
||||||
@@ -483,7 +525,7 @@ func MeshConsumers() []Consumer {
|
|||||||
Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
|
Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
|
||||||
"because a notice it gave up on would lose the message it is about",
|
"because a notice it gave up on would lose the message it is about",
|
||||||
},
|
},
|
||||||
}
|
}, CheckWorkers()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330).
|
// NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330).
|
||||||
|
|||||||
@@ -134,6 +134,9 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
|
|||||||
// What each machine's provisioner client last said of each module's files (design 57 P6): the newest
|
// What each machine's provisioner client last said of each module's files (design 57 P6): the newest
|
||||||
// word per module, what a walk's end reads.
|
// word per module, what a walk's end reads.
|
||||||
"PROV_APPLIED": RetentionLastPerSubject,
|
"PROV_APPLIED": RetentionLastPerSubject,
|
||||||
|
// The checks of pull requests, in two work queues of their own (novox/hq ADR 0406 §3, issue 974).
|
||||||
|
"CHECKS_READY": RetentionWorkQueue,
|
||||||
|
"CHECKS_OTHER": RetentionWorkQueue,
|
||||||
}
|
}
|
||||||
got := map[string]Retention{}
|
got := map[string]Retention{}
|
||||||
for _, s := range MeshStreams() {
|
for _, s := range MeshStreams() {
|
||||||
|
|||||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
|||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
|
|||||||
@@ -102,7 +102,8 @@ type Records struct {
|
|||||||
// is a mesh that cannot be told anything, and there is no state of the records in which that is
|
// is a mesh that cannot be told anything, and there is no state of the records in which that is
|
||||||
// correct.
|
// correct.
|
||||||
func Users(r Records) ([]Principal, error) {
|
func Users(r Records) ([]Principal, error) {
|
||||||
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r)}}
|
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r),
|
||||||
|
AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig)}}
|
||||||
|
|
||||||
for _, node := range sortedCopy(r.Nodes) {
|
for _, node := range sortedCopy(r.Nodes) {
|
||||||
witness := false
|
witness := false
|
||||||
@@ -290,13 +291,16 @@ func holdsSeat(d Declared, seat string) bool {
|
|||||||
|
|
||||||
// vaultVerbsOf are the vault's intake verbs its holder serves, as the records say (issue 675): what the controller
|
// vaultVerbsOf are the vault's intake verbs its holder serves, as the records say (issue 675): what the controller
|
||||||
// is granted to ask it in its own name. None while no holder serves them.
|
// is granted to ask it in its own name. None while no holder serves them.
|
||||||
func vaultVerbsOf(r Records) []string {
|
func vaultVerbsOf(r Records) []string { return servedOf(r, VerbsTheControllerAsksTheVault) }
|
||||||
|
|
||||||
|
// servedOf are the verbs of asked that a seat's holder serves, as the records say, by name.
|
||||||
|
func servedOf(r Records, asked []SeatVerb) []string {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
var out []string
|
var out []string
|
||||||
for _, node := range sortedCopy(r.Nodes) {
|
for _, node := range sortedCopy(r.Nodes) {
|
||||||
for _, d := range r.Assigned[node] {
|
for _, d := range r.Assigned[node] {
|
||||||
for _, s := range d.Holds {
|
for _, s := range d.Holds {
|
||||||
for _, v := range VerbsTheControllerAsksTheVault {
|
for _, v := range asked {
|
||||||
if s.Name == v.Seat && slices.Contains(s.Serves, v.Verb) && !seen[v.Verb] {
|
if s.Name == v.Seat && slices.Contains(s.Serves, v.Verb) && !seen[v.Verb] {
|
||||||
seen[v.Verb] = true
|
seen[v.Verb] = true
|
||||||
out = append(out, v.Verb)
|
out = append(out, v.Verb)
|
||||||
|
|||||||
@@ -41,6 +41,14 @@ func ownMachine(p Principal, pattern string) bool {
|
|||||||
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
|
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// holdsTheBuildSeat is a principal holding the build seat, or the controller, which asserts the bucket the holders write.
|
||||||
|
func holdsTheBuildSeat(p Principal, _ string) bool {
|
||||||
|
if p.Kind == KindController {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return holdsSeatOf(p, "mesh.seat."+TheBuildSeat+".x")
|
||||||
|
}
|
||||||
|
|
||||||
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
|
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
|
||||||
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
|
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
|
||||||
func holdsSeatOf(p Principal, pattern string) bool {
|
func holdsSeatOf(p Principal, pattern string) bool {
|
||||||
@@ -117,8 +125,13 @@ var WritersTable = []WriterRow{
|
|||||||
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine},
|
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine},
|
||||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||||
|
// And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1), and the
|
||||||
|
// wait of a rollout for each machine it sent (ADR 0406 §5, issue 974): keys that expire in buckets of their own,
|
||||||
|
// written by the lease holder.
|
||||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||||
KeptIn: "the controller's store", Others: "read through plans"},
|
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket +
|
||||||
|
", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket, Others: "read through plans",
|
||||||
|
Subjects: append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), Writes: isController},
|
||||||
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
||||||
ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
|
ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
|
||||||
"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
|
"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
|
||||||
@@ -143,6 +156,15 @@ var WritersTable = []WriterRow{
|
|||||||
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
|
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
|
||||||
Writes: holdsSeatOf,
|
Writes: holdsSeatOf,
|
||||||
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
|
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
|
||||||
|
{State: "a pull request's check asked", Writer: "controller",
|
||||||
|
KeptIn: "the work queues " + ChecksReadyStream + " and " + ChecksOtherStream + ", one subject a pull request, the newest kept",
|
||||||
|
Others: "the build seat's holders take it",
|
||||||
|
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">", CheckSubjectPrefix(TheBuildSeat, false) + ">"},
|
||||||
|
Writes: isController},
|
||||||
|
{State: "a pull request's check running", Writer: "the build seat's holders — several by design, each the key of the check it runs",
|
||||||
|
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
|
||||||
|
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
|
||||||
|
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
|
||||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||||
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
|
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ var designRows = []string{
|
|||||||
"the healers' acts and their brake",
|
"the healers' acts and their brake",
|
||||||
"stream definitions and bus permissions",
|
"stream definitions and bus permissions",
|
||||||
"builds and their outcomes",
|
"builds and their outcomes",
|
||||||
|
"a pull request's check asked",
|
||||||
|
"a pull request's check running",
|
||||||
"a merge announced",
|
"a merge announced",
|
||||||
"a pull request's head announced",
|
"a pull request's head announced",
|
||||||
"a pull request's merge check",
|
"a pull request's merge check",
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
|
|||||||
}
|
}
|
||||||
held.From, held.At, held.Module = node.Name, at, m.Module
|
held.From, held.At, held.Module = node.Name, at, m.Module
|
||||||
held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name)
|
held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name)
|
||||||
held.CredentialForm = m.CredentialFormOf(n.Name)
|
held.CredentialForm, held.CredentialTool = m.CredentialFormOf(n.Name), m.CredentialToolOf(n.Name)
|
||||||
return held, true, ""
|
return held, true, ""
|
||||||
}
|
}
|
||||||
matching := bound.among(world.Offered[n.Name])
|
matching := bound.among(world.Offered[n.Name])
|
||||||
@@ -125,7 +125,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
|
|||||||
identity := DefaultIdentityBound
|
identity := DefaultIdentityBound
|
||||||
if pm, known := catalogue[p.Module]; known {
|
if pm, known := catalogue[p.Module]; known {
|
||||||
held.SharedOwn, _ = pm.SharedCredentialOf(n.Name)
|
held.SharedOwn, _ = pm.SharedCredentialOf(n.Name)
|
||||||
held.CredentialForm = pm.CredentialFormOf(n.Name)
|
held.CredentialForm, held.CredentialTool = pm.CredentialFormOf(n.Name), pm.CredentialToolOf(n.Name)
|
||||||
identity = pm.IdentityBoundOf(n.Name)
|
identity = pm.IdentityBoundOf(n.Name)
|
||||||
}
|
}
|
||||||
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
|
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node's files, read and managed (novox/hq ADR 0394, issue 900; superseding ADR 0310; hq to-be 41 WP9).
|
||||||
|
//
|
||||||
|
// **One seat for every file on a node.** The holder runs as the tool runner's account, the node's operator
|
||||||
|
// account (ADR 0394 §2), and reaches further through `sudo -n` where that account's rights do not. It refuses by
|
||||||
|
// name the places a secret is kept (reads only, ADR 0394 §7) and the paths a module owns, unless a write names a reason (`override`).
|
||||||
|
// node-reads stays beside it until mesh-files is assigned on every node (ADR 0394 §9); it goes in the next
|
||||||
|
// delivery.
|
||||||
|
|
||||||
|
// FilesSeat is the role that reads and manages a node's files and accounts (novox/hq ADR 0394).
|
||||||
|
const FilesSeat = "node-files"
|
||||||
|
|
||||||
|
// FilesOwnedPaths is the word the holder of node-files is given beside the secret places: a JSON object from
|
||||||
|
// each absolute path a module owns on its node to "module (tool to use)" (novox/hq ADR 0394 §5).
|
||||||
|
const FilesOwnedPaths = "MESH_OWNED_PATHS"
|
||||||
|
|
||||||
|
const (
|
||||||
|
filesAsAccount = "as the node's operator account, through sudo -n where its rights do not reach; a place a " +
|
||||||
|
"secret is kept, or a known credential file, is refused by name"
|
||||||
|
overrideWord = "a reason, to change a path a module owns anyway (optional)"
|
||||||
|
)
|
||||||
|
|
||||||
|
// filesVerbs is the contract every holder of node-files serves (novox/hq ADR 0394 §1): the reads of node-reads,
|
||||||
|
// with their descriptions restated for the operator's account, and the writes.
|
||||||
|
func filesVerbs() []Verb {
|
||||||
|
var verbs []Verb
|
||||||
|
for _, v := range readsVerbs() {
|
||||||
|
v.Description = strings.Replace(v.Description,
|
||||||
|
"Read as the holder's own account, which holds no secret: a component it may not reach says so.",
|
||||||
|
"", 1)
|
||||||
|
v.Description = strings.TrimSpace(v.Description) + " Read " + filesAsAccount + "."
|
||||||
|
verbs = append(verbs, v)
|
||||||
|
}
|
||||||
|
write := func(name, description string, in map[string]string, required []string, optional []string, replaces ...string) Verb {
|
||||||
|
in["reason"] = "why the file is changed, for the audit trail"
|
||||||
|
in["override"] = overrideWord
|
||||||
|
return Verb{Name: name, Description: description + " Refuses a path a module owns " +
|
||||||
|
"unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.",
|
||||||
|
Input: schema(in, append(required, "reason"), append(optional, "override")...), Replaces: replaces}
|
||||||
|
}
|
||||||
|
return append(verbs,
|
||||||
|
write("write", "Write one text file whole, replacing what it holds; the old content goes to the node's trash.",
|
||||||
|
map[string]string{"path": "an absolute path", "content": "the text to write",
|
||||||
|
"mode": "the file's mode, such as 0644 (optional)"},
|
||||||
|
[]string{"path", "content"}, []string{"mode"}, "tee", ">"),
|
||||||
|
write("copy", "Copy a file or directory to a path on this node. A target on another node (<node>:/path) is "+
|
||||||
|
"refused: copying between nodes comes in step 2 (issue 903). An existing target is overwritten, the old one "+
|
||||||
|
"kept in the node's trash for 7 days.",
|
||||||
|
map[string]string{"from": "an absolute path", "to": "an absolute path on this node"},
|
||||||
|
[]string{"from", "to"}, nil, "cp", "cp -a", "ssh <node> cp"),
|
||||||
|
write("move", "Move or rename a file or directory. A target on another node is refused: copying between nodes "+
|
||||||
|
"comes in step 2 (issue 903). An existing target is overwritten, the old one kept in the node's trash for 7 days.",
|
||||||
|
map[string]string{"from": "an absolute path", "to": "an absolute path"},
|
||||||
|
[]string{"from", "to"}, nil, "mv"),
|
||||||
|
write("delete", "Delete a file or directory into the node's trash, where restore can bring it back.",
|
||||||
|
map[string]string{"path": "an absolute path", "recursive": "\"true\": delete a directory with what it holds"},
|
||||||
|
[]string{"path"}, []string{"recursive"}, "rm", "rm -r"),
|
||||||
|
write("mkdir", "Make a directory.",
|
||||||
|
map[string]string{"path": "an absolute path", "parents": "\"true\": make the missing parents too"},
|
||||||
|
[]string{"path"}, []string{"parents"}, "mkdir", "mkdir -p"),
|
||||||
|
write("chmod", "Change the mode of a file or directory.",
|
||||||
|
map[string]string{"path": "an absolute path", "mode": "the mode, such as 0644 or u+x",
|
||||||
|
"recursive": "\"true\": also everything under a directory"},
|
||||||
|
[]string{"path", "mode"}, []string{"recursive"}, "chmod"),
|
||||||
|
write("chown", "Change the owner and group of a file or directory.",
|
||||||
|
map[string]string{"path": "an absolute path", "owner": "an account's name or number",
|
||||||
|
"group": "a group's name or number (optional)", "recursive": "\"true\": also everything under a directory"},
|
||||||
|
[]string{"path", "owner"}, []string{"group", "recursive"}, "chown", "chgrp"),
|
||||||
|
Verb{Name: "trash", Description: "What the node's trash holds: each deleted or replaced item with its id, " +
|
||||||
|
"original path and when it went.",
|
||||||
|
Input: schema(map[string]string{}, nil), Replaces: []string{"ls ~/.local/share/Trash"}},
|
||||||
|
write("restore", "Put an item of the node's trash back where it was; refuses when something is there now.",
|
||||||
|
map[string]string{"id": "an id from trash"}, []string{"id"}, nil, "trash-restore"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FilesHolderProblems is what a holder of node-files is held to (novox/hq ADR 0394 §2): it runs as the tool
|
||||||
|
// runner's account, the node's operator account, so it declares no runs-as.
|
||||||
|
func FilesHolderProblems(m Manifest) []string {
|
||||||
|
if !m.ClaimsSeat(FilesSeat) || m.RunsAs == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []string{fmt.Sprintf("%s claims %s and declares runs-as %q: its holder runs as the tool runner's account, "+
|
||||||
|
"the node's operator account (novox/hq ADR 0394 §2)", m.Module, FilesSeat, m.RunsAs)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownedPaths is every path a module owns on this node, as the holder is told (novox/hq ADR 0394 §5): the
|
||||||
|
// paths every seat declares (ADR 0287) with the module holding the seat here, and the paths and files each
|
||||||
|
// module's manifest places (declaredPaths). A seat's path wins where both name one.
|
||||||
|
func (r Resolution) ownedPaths() map[string]string {
|
||||||
|
owned := map[string]string{}
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
for _, p := range sortedKeys(declaredPaths(m)) {
|
||||||
|
owned[p] = m.Module + " (its own tools)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range Seats() {
|
||||||
|
for _, sp := range s.Paths {
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
if m.ClaimsSeat(s.Name) && declaresPath(m, path.Clean(sp.Path)) {
|
||||||
|
owned[sp.Path] = m.Module + " (" + s.Name + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return owned
|
||||||
|
}
|
||||||
|
|
||||||
|
// filesWords is what the holder of node-files is given beside its own words: the node's secret places and the
|
||||||
|
// paths modules own here.
|
||||||
|
func (r Resolution) filesWords(m Manifest, words map[string]string, with Rendering) (map[string]string, error) {
|
||||||
|
if !m.ClaimsSeat(FilesSeat) {
|
||||||
|
return words, nil
|
||||||
|
}
|
||||||
|
places, err := r.secretPlaces(with)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
places2, err := json.Marshal(places)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
owned, err := json.Marshal(r.ownedPaths())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if words == nil {
|
||||||
|
words = map[string]string{}
|
||||||
|
}
|
||||||
|
for _, w := range []string{ReadsSecretPlaces, FilesOwnedPaths} {
|
||||||
|
if _, has := words[w]; has {
|
||||||
|
return nil, fmt.Errorf("%s gives its own %s; the node's secret places and owned paths are the mesh's to "+
|
||||||
|
"say (novox/hq ADR 0394 §5)", m.Module, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
words[ReadsSecretPlaces] = string(places2)
|
||||||
|
words[FilesOwnedPaths] = string(owned)
|
||||||
|
return words, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// issue 900, novox/hq ADR 0394 §1: node-files is a node seat whose verbs say the commands they replace.
|
||||||
|
func TestNodeFilesIsANodeSeatWithEveryVerbOfADR0394(t *testing.T) {
|
||||||
|
s, ok := SeatNamed(FilesSeat)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("the mesh defines no seat %s", FilesSeat)
|
||||||
|
}
|
||||||
|
if s.Scope != ScopeNode || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 0394") {
|
||||||
|
t.Errorf("%s: scope %q, delivers %q, decision %q", FilesSeat, s.Scope, s.Delivers, s.Decision)
|
||||||
|
}
|
||||||
|
have := map[string]bool{}
|
||||||
|
replaced := map[string]string{}
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
have[v.Name] = true
|
||||||
|
if v.Optional {
|
||||||
|
t.Errorf("%s.%s is optional", FilesSeat, v.Name)
|
||||||
|
}
|
||||||
|
if len(v.Replaces) == 0 {
|
||||||
|
t.Errorf("%s.%s says nothing it replaces", FilesSeat, v.Name)
|
||||||
|
}
|
||||||
|
for _, c := range v.Replaces {
|
||||||
|
replaced[c] = v.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range []string{"stat", "readlink", "list", "read", "search", "account", "write", "copy", "move",
|
||||||
|
"delete", "mkdir", "chmod", "chown", "trash", "restore"} {
|
||||||
|
if !have[name] {
|
||||||
|
t.Errorf("%s serves no verb %s", FilesSeat, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for command, verb := range map[string]string{"cat": "read", "tee": "write", "cp": "copy", "mv": "move",
|
||||||
|
"rm": "delete", "mkdir -p": "mkdir", "chmod": "chmod", "chgrp": "chown"} {
|
||||||
|
if replaced[command] != verb {
|
||||||
|
t.Errorf("%q is replaced by %q, want %s.%s", command, replaced[command], FilesSeat, verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// issue 900, ADR 0394 §2: a holder of node-files runs as the tool runner's account, so it declares no runs-as.
|
||||||
|
func TestAHolderOfNodeFilesDeclaringRunsAsIsRefused(t *testing.T) {
|
||||||
|
p := FilesHolderProblems(Manifest{Module: "files", RunsAs: "mesh-files", Claims: []Claim{{Name: FilesSeat}}})
|
||||||
|
if len(p) != 1 || !strings.Contains(p[0], "0394") || !strings.Contains(p[0], "runs-as") {
|
||||||
|
t.Errorf("a holder declaring runs-as was not refused: %v", p)
|
||||||
|
}
|
||||||
|
if p := FilesHolderProblems(Manifest{Module: "files", Claims: []Claim{{Name: FilesSeat}}}); len(p) != 0 {
|
||||||
|
t.Errorf("a holder declaring none was refused: %v", p)
|
||||||
|
}
|
||||||
|
if p := FilesHolderProblems(Manifest{Module: "other", RunsAs: "x"}); len(p) != 0 {
|
||||||
|
t.Errorf("a non-holder was judged: %v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// issue 900, ADR 0394 §4: the holder is told the secret places and the paths modules own on its node.
|
||||||
|
func TestTheHolderOfNodeFilesIsToldSecretPlacesAndOwnedPaths(t *testing.T) {
|
||||||
|
files := Manifest{Module: "files", Version: "1", Claims: []Claim{{Name: FilesSeat}},
|
||||||
|
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/files/broker"}}}
|
||||||
|
host := Manifest{Module: "hostnames", Version: "1", Claims: []Claim{{Name: "node-hostname"}},
|
||||||
|
Resources: []map[string]any{{"id": "hosts", "type": "file", "path": "/etc/hosts"}}}
|
||||||
|
app := Manifest{Module: "app", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/app/app.conf"}}}
|
||||||
|
r := Resolution{Node: "anchor", Modules: []Manifest{files, host, app}}
|
||||||
|
words, err := r.filesWords(files, nil, Rendering{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var places []string
|
||||||
|
if err := json.Unmarshal([]byte(words[ReadsSecretPlaces]), &places); err != nil || len(places) == 0 {
|
||||||
|
t.Errorf("no secret places: %v %v", err, words)
|
||||||
|
}
|
||||||
|
var owned map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(words[FilesOwnedPaths]), &owned); err != nil {
|
||||||
|
t.Fatalf("no owned paths: %v (%v)", err, words)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(owned["/etc/hosts"], "hostnames") {
|
||||||
|
t.Errorf("/etc/hosts is owned by %q, want hostnames", owned["/etc/hosts"])
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(owned["/etc/app/app.conf"], "app") {
|
||||||
|
t.Errorf("a placed file is owned by %q, want app", owned["/etc/app/app.conf"])
|
||||||
|
}
|
||||||
|
if w, _ := r.filesWords(app, nil, Rendering{}); w[FilesOwnedPaths] != "" {
|
||||||
|
t.Error("a module not holding node-files is told the owned paths")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// issue 900: restore, like every write verb, takes a reason and an override.
|
||||||
|
func TestNodeFilesRestoreTakesIdAndReason(t *testing.T) {
|
||||||
|
s, _ := SeatNamed(FilesSeat)
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
if v.Name != "restore" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
props, _ := v.Input["properties"].(map[string]any)
|
||||||
|
for _, p := range []string{"id", "reason", "override"} {
|
||||||
|
if _, has := props[p]; !has {
|
||||||
|
t.Errorf("restore has no input %s", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req, _ := v.Input["required"].([]string); strings.Join(req, ",") != "id,reason" {
|
||||||
|
t.Errorf("restore requires %v, want id and reason", req)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Error("no restore verb")
|
||||||
|
}
|
||||||
@@ -104,6 +104,9 @@ type PlanProvision struct {
|
|||||||
Provider string `json:"provider"`
|
Provider string `json:"provider"`
|
||||||
ProviderModule string `json:"provider-module"`
|
ProviderModule string `json:"provider-module"`
|
||||||
CredentialForm string `json:"credential-form,omitempty"`
|
CredentialForm string `json:"credential-form,omitempty"`
|
||||||
|
// CredentialTool is, for an offer whose credential the provider's software issues (issue 928), the provider's tool
|
||||||
|
// the vault asks for each holder's value; empty for any other form.
|
||||||
|
CredentialTool string `json:"credential-tool,omitempty"`
|
||||||
Path string `json:"path,omitempty"`
|
Path string `json:"path,omitempty"`
|
||||||
// Contributes is what the consumer's manifest tells its provider for this provision (Manifest.Contributes: a
|
// Contributes is what the consumer's manifest tells its provider for this provision (Manifest.Contributes: a
|
||||||
// database's name, its extensions), so the provider's helper learns it from the bus (issue 776). Nil where the
|
// database's name, its extensions), so the provider's helper learns it from the bus (issue 776). Nil where the
|
||||||
@@ -134,7 +137,7 @@ func provisionsOf(needs []Needed, module, as string, contributes map[string]map[
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
p := PlanProvision{Provision: n.Name, Local: n.Local, Provider: n.From, ProviderModule: n.Module,
|
p := PlanProvision{Provision: n.Name, Local: n.Local, Provider: n.From, ProviderModule: n.Module,
|
||||||
CredentialForm: n.CredentialForm, Contributes: maps.Clone(contributes[n.Name]),
|
CredentialForm: n.CredentialForm, CredentialTool: n.CredentialTool, Contributes: maps.Clone(contributes[n.Name]),
|
||||||
At: n.At, From: n.From, As: as}
|
At: n.At, From: n.From, As: as}
|
||||||
// **A credential only a template names** (issue 788): named like a file's, so the client fills it from the
|
// **A credential only a template names** (issue 788): named like a file's, so the client fills it from the
|
||||||
// vault's row for this holder.
|
// vault's row for this holder.
|
||||||
|
|||||||
@@ -123,3 +123,37 @@ func TestAHandedModulesLocalCredentialsOfTheVaultAreNamedByLocalInItsPlan(t *tes
|
|||||||
t.Fatalf("the plan names %v, want admin and internal-token", names)
|
t.Fatalf("the plan names %v, want admin and internal-token", names)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **Issue 969: a key its provider serves only at run time.** A moved module's file may name `${bound:…}` of a
|
||||||
|
// provision whose provider tells it at run time (ssh-client's known_host of the forge). Where the machine takes
|
||||||
|
// handed files the client fills it and the module composes; where it does not the controller must fill it from what
|
||||||
|
// the provider serves statically, cannot, and refuses it with the very words an unmarked module is refused with
|
||||||
|
// (ADR 0349 §17: the old way, byte for byte).
|
||||||
|
func TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded(t *testing.T) {
|
||||||
|
text := "Host ${bound:postgres-database:at}\n KnownHost ${bound:postgres-database:known_host}\n"
|
||||||
|
moved := strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`,
|
||||||
|
`"requires":["postgres-database"],"own-secrets":{"token":"${dir:state}/token"}`, 1)
|
||||||
|
unmoved := strings.Replace(moved, `"moved":true,`, ``, 1)
|
||||||
|
if unmoved == moved {
|
||||||
|
t.Fatal("the twin is still marked moved")
|
||||||
|
}
|
||||||
|
|
||||||
|
c, got := composeWith(t, withDatabase(t, moved, text, true), handingTo("store:5000"))
|
||||||
|
if strings.Contains(got, "known_host") || len(c.Filled) == 0 {
|
||||||
|
t.Fatalf("handed, the module is filled by the client and the controller names nothing: %v %s", c.Filled, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, with := range map[string]Rendering{
|
||||||
|
"an engine that does not take a filled file alone": {Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
|
||||||
|
"no provisioner client on the machine": {Hands: true, ArtifactStore: "store:5000"},
|
||||||
|
} {
|
||||||
|
_, err := withDatabase(t, moved, text, true).Compose(sealedForBot(with))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `asks its postgres-database binding for "known_host"`) {
|
||||||
|
t.Fatalf("%s: the key is not refused: %v", name, err)
|
||||||
|
}
|
||||||
|
_, before := withDatabase(t, unmoved, text, false).Compose(sealedForBot(with))
|
||||||
|
if before == nil || before.Error() != err.Error() {
|
||||||
|
t.Fatalf("%s: refused otherwise than the unmarked module: %v / %v", name, err, before)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -242,3 +242,33 @@ func TestAnOfferAnsweredByTheHelperIsReadAndHandsNoFile(t *testing.T) {
|
|||||||
t.Fatalf("the flag changed what is composed or handed:\nbefore: %s\nnow: %s\n%v %v", before, got, c2.Filled, c2.Plans)
|
t.Fatalf("the flag changed what is composed or handed:\nbefore: %s\nnow: %s\n%v %v", before, got, c2.Filled, c2.Plans)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A plan entry names the tool of an offer whose credential the provider's software issues (issue 928), so the vault
|
||||||
|
// asks that tool for each holder; an offer of any other form names none.
|
||||||
|
func TestAHandedModulesPlanNamesTheCredentialToolOfASoftwareIssuedOffer(t *testing.T) {
|
||||||
|
for credential, want := range map[string]string{
|
||||||
|
``: "",
|
||||||
|
`,"credential":{"own":"admin"}`: "",
|
||||||
|
`,"credential":{"issued-by":"outside"}`: "",
|
||||||
|
`,"credential":{"issued-by":"software","tool":"gitea_agent_token"}`: "gitea_agent_token",
|
||||||
|
} {
|
||||||
|
raw := `{"module":"store","version":"1","provides":[{"name":"postgres-database","scope":"mesh"` +
|
||||||
|
credential + `}],"serves":{"postgres-database":{"port":5432}}`
|
||||||
|
if strings.Contains(credential, `"own"`) {
|
||||||
|
raw += `,"own-secrets":{"admin":{"path":"/var/lib/store/admin","taken":"at-start"}}`
|
||||||
|
}
|
||||||
|
node := workstation()
|
||||||
|
node.At = "workstation.internal"
|
||||||
|
consumer := notifierConsumer(t, true, false)
|
||||||
|
r, err := Resolve(shelf(consumer, mustParse(t, raw+`}`)), []string{consumer.Module}, node, twoStores("anchor"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r.Account = "op"
|
||||||
|
c, _ := composeWith(t, r, handingTo("store:5000"))
|
||||||
|
got := c.Plans["notify"].Provisions
|
||||||
|
if len(got) != 1 || got[0].CredentialTool != want {
|
||||||
|
t.Errorf("an offer with %q: the plan names %+v, want the tool %q", credential, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A jail whose pattern names <HOST> twice is refused by name: fail2ban would read it as a duplicate
|
||||||
|
// capture group and refuse its whole configuration, leaving the machine with no bans at all — which
|
||||||
|
// is what happened on the control node the day this was built (novox/hq ADR 0179).
|
||||||
|
func TestAJailsPatternNamesTheHostOncePerShape(t *testing.T) {
|
||||||
|
twice := Manifest{Module: "route-proxy", Jails: []Jail{{
|
||||||
|
Name: "route-proxy",
|
||||||
|
Failregex: `^.*(?:refused for <HOST>:\d+|asked from <HOST>:\d+)$`,
|
||||||
|
Jail: "backend = systemd",
|
||||||
|
}}}
|
||||||
|
problems := twice.jailProblems()
|
||||||
|
if len(problems) != 1 || !strings.Contains(problems[0], "names <HOST> 2 times") {
|
||||||
|
t.Fatalf("a pattern naming <HOST> twice was not refused by name: %v", problems)
|
||||||
|
}
|
||||||
|
if !strings.Contains(problems[0], "no bans at all") {
|
||||||
|
t.Errorf("the refusal does not say what it costs: %q", problems[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same two shapes, one pattern each, is how it is written — and is accepted.
|
||||||
|
split := Manifest{Module: "route-proxy", Jails: []Jail{{
|
||||||
|
Name: "route-proxy",
|
||||||
|
Failregex: "^.*refused for <HOST>:\\d+$\n ^.*asked from <HOST>:\\d+$",
|
||||||
|
Jail: "backend = systemd",
|
||||||
|
}}}
|
||||||
|
if problems := split.jailProblems(); len(problems) != 0 {
|
||||||
|
t.Fatalf("one <HOST> per pattern was refused: %v", problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A jail with no name, no pattern, or a name another jail of the same module already took is refused.
|
||||||
|
func TestAJailIsNamedOnceAndSaysWhatAFailureLooksLike(t *testing.T) {
|
||||||
|
m := Manifest{Module: "mailu", Jails: []Jail{
|
||||||
|
{Name: "", Failregex: "^.*rip=<HOST>$"},
|
||||||
|
{Name: "mailu-front", Failregex: " "},
|
||||||
|
{Name: "mailu-front", Failregex: "^.*rip=<HOST>$"},
|
||||||
|
}}
|
||||||
|
problems := strings.Join(m.jailProblems(), "\n")
|
||||||
|
for _, want := range []string{"a jail with no name", "says nothing a failed attempt looks like", "two jails called"} {
|
||||||
|
if !strings.Contains(problems, want) {
|
||||||
|
t.Errorf("missing %q in:\n%s", want, problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -303,6 +303,13 @@ func (m Manifest) IssuedCredentialOf(provision string) (string, bool) {
|
|||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CredentialToolOf is the tool a plan names for a provision, set only where the offer says the provider's software
|
||||||
|
// issues its credential (issue 928); empty otherwise.
|
||||||
|
func (m Manifest) CredentialToolOf(provision string) string {
|
||||||
|
tool, _ := m.IssuedCredentialOf(provision)
|
||||||
|
return tool
|
||||||
|
}
|
||||||
|
|
||||||
// IssuedOutsideOf is whether this module's offer of a provision says only a person outside the mesh can give its
|
// IssuedOutsideOf is whether this module's offer of a provision says only a person outside the mesh can give its
|
||||||
// credential (novox/hq ADR 0337): the mesh never makes a value for it, and a consumer holds one only once a person
|
// credential (novox/hq ADR 0337): the mesh never makes a value for it, and a consumer holds one only once a person
|
||||||
// gave it with `secret accept`.
|
// gave it with `secret accept`.
|
||||||
@@ -1984,6 +1991,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
|
|||||||
problems = append(problems, declaredSeatProblems(m)...)
|
problems = append(problems, declaredSeatProblems(m)...)
|
||||||
problems = append(problems, RunsAsProblems(m)...)
|
problems = append(problems, RunsAsProblems(m)...)
|
||||||
problems = append(problems, ReadsHolderProblems(m)...)
|
problems = append(problems, ReadsHolderProblems(m)...)
|
||||||
|
problems = append(problems, FilesHolderProblems(m)...)
|
||||||
if m.Computed != "" && len(m.Resources) > 0 {
|
if m.Computed != "" && len(m.Resources) > 0 {
|
||||||
// One or the other. A module that both ships files and has them computed would leave
|
// One or the other. A module that both ships files and has them computed would leave
|
||||||
// nobody able to say where a given file came from.
|
// nobody able to say where a given file came from.
|
||||||
|
|||||||
@@ -230,6 +230,8 @@ type Needed struct {
|
|||||||
// CredentialForm is the credential form the provider's offer declares (Manifest.CredentialFormOf, issue 768),
|
// CredentialForm is the credential form the provider's offer declares (Manifest.CredentialFormOf, issue 768),
|
||||||
// named in a handed consumer's plan for the vault; empty where the provider's manifest is not in hand.
|
// named in a handed consumer's plan for the vault; empty where the provider's manifest is not in hand.
|
||||||
CredentialForm string
|
CredentialForm string
|
||||||
|
// CredentialTool is the tool of the offer that the provider's software issues the credential through (issue 928).
|
||||||
|
CredentialTool string
|
||||||
// Manager is set when this holder is a refreshable-grant licence's MANAGER, delivered the refresh
|
// Manager is set when this holder is a refreshable-grant licence's MANAGER, delivered the refresh
|
||||||
// token rather than an access token (novox/hq ADR 0050). It changes one thing downstream: an empty
|
// token rather than an access token (novox/hq ADR 0050). It changes one thing downstream: an empty
|
||||||
// Sealed is tolerated — the manager has not adopted a refresh token yet, which is a real waiting
|
// Sealed is tolerated — the manager has not adopted a refresh token yet, which is a real waiting
|
||||||
@@ -543,7 +545,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
needs = append(needs, Needed{
|
needs = append(needs, Needed{
|
||||||
Name: want, From: node.Name, At: at, Module: by.Module,
|
Name: want, From: node.Name, At: at, Module: by.Module,
|
||||||
Serves: servedByOne(by, want), For: because[want],
|
Serves: servedByOne(by, want), For: because[want],
|
||||||
SharedOwn: sharedByOne(by, want), CredentialForm: by.CredentialFormOf(want),
|
SharedOwn: sharedByOne(by, want), CredentialForm: by.CredentialFormOf(want), CredentialTool: by.CredentialToolOf(want),
|
||||||
Identity: by.IdentityBoundOf(want)})
|
Identity: by.IdentityBoundOf(want)})
|
||||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||||
// Answered here with no credential to mint, but the provider serves facts the
|
// Answered here with no credential to mint, but the provider serves facts the
|
||||||
@@ -565,7 +567,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
}
|
}
|
||||||
needs = append(needs, Needed{
|
needs = append(needs, Needed{
|
||||||
Name: want, From: node.Name, At: at, Serves: served, For: because[want], Module: by.Module,
|
Name: want, From: node.Name, At: at, Serves: served, For: because[want], Module: by.Module,
|
||||||
CredentialForm: by.CredentialFormOf(want)})
|
CredentialForm: by.CredentialFormOf(want), CredentialTool: by.CredentialToolOf(want)})
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -594,17 +596,17 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
node.Name, want, p.Node, meshNetwork))
|
node.Name, want, p.Node, meshNetwork))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
shared, form := "", ""
|
shared, form, tool := "", "", ""
|
||||||
// A provider whose definition is not in hand is held to the tightest bound the
|
// A provider whose definition is not in hand is held to the tightest bound the
|
||||||
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||||
bound := DefaultIdentityBound
|
bound := DefaultIdentityBound
|
||||||
if pm, known := catalogue[p.Module]; known {
|
if pm, known := catalogue[p.Module]; known {
|
||||||
shared, _ = pm.SharedCredentialOf(want)
|
shared, _ = pm.SharedCredentialOf(want)
|
||||||
form = pm.CredentialFormOf(want)
|
form, tool = pm.CredentialFormOf(want), pm.CredentialToolOf(want)
|
||||||
bound = pm.IdentityBoundOf(want)
|
bound = pm.IdentityBoundOf(want)
|
||||||
}
|
}
|
||||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
|
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
|
||||||
Serves: p.Serves, For: because[want], SharedOwn: shared, CredentialForm: form, Identity: bound,
|
Serves: p.Serves, For: because[want], SharedOwn: shared, CredentialForm: form, CredentialTool: tool, Identity: bound,
|
||||||
LeftOut: p.LeftOut})
|
LeftOut: p.LeftOut})
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -178,6 +178,10 @@ func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// The holder of node-files is told the same, and the paths modules own here (novox/hq ADR 0394 §4).
|
||||||
|
if words, err = r.filesWords(m, words, with); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if len(words) > 0 {
|
if len(words) > 0 {
|
||||||
given[m.Module] = words
|
given[m.Module] = words
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -108,6 +108,8 @@ var defaultSeats = append([]Seat{
|
|||||||
"checked",
|
"checked",
|
||||||
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||||
"plan-moved",
|
"plan-moved",
|
||||||
|
// A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||||
|
"sent",
|
||||||
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||||
"assignment-changed", "declared-config-changed", "predefined-changed"},
|
"assignment-changed", "declared-config-changed", "predefined-changed"},
|
||||||
Serves: ControllerVerbs},
|
Serves: ControllerVerbs},
|
||||||
@@ -421,6 +423,9 @@ var defaultSeats = append([]Seat{
|
|||||||
// A node's files and accounts, read (novox/hq ADR 0310, reads.go): by an account of the holder's own that holds
|
// A node's files and accounts, read (novox/hq ADR 0310, reads.go): by an account of the holder's own that holds
|
||||||
// no secret, on every node, so the reads ADR 0268 §5 takes over from the login shell's execute have a tool.
|
// no secret, on every node, so the reads ADR 0268 §5 takes over from the login shell's execute have a tool.
|
||||||
{Name: ReadsSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0310", Serves: readsVerbs()},
|
{Name: ReadsSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0310", Serves: readsVerbs()},
|
||||||
|
// A node's files, read and managed (novox/hq ADR 0394, files.go): beside node-reads until mesh-files is assigned
|
||||||
|
// on every node, then node-reads goes (ADR 0394 §9).
|
||||||
|
{Name: FilesSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0394", Serves: filesVerbs()},
|
||||||
},
|
},
|
||||||
// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
|
// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
|
||||||
graphicalSessionSeats()...)
|
graphicalSessionSeats()...)
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
|||||||
delivered[s.Delivers] = s.Name
|
delivered[s.Delivers] = s.Name
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Forty-five with node-files (novox/hq ADR 0394); forty-four again once node-reads goes.
|
||||||
// Forty-four with node-provisioner (novox/hq ADR 0349 clause 3); forty-three with mesh-provisioner (novox/hq ADR 0349 clause 3); forty-two with mesh-config (novox/hq ADR 0348 §9); forty-one with node-reads (novox/hq ADR 0310); forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
// Forty-four with node-provisioner (novox/hq ADR 0349 clause 3); forty-three with mesh-provisioner (novox/hq ADR 0349 clause 3); forty-two with mesh-config (novox/hq ADR 0348 §9); forty-one with node-reads (novox/hq ADR 0310); forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
||||||
// into node-uplink (novox/hq ADR 0223); thirty-seven
|
// into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||||
@@ -59,8 +60,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
|||||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||||
if len(Seats()) != 44 {
|
if len(Seats()) != 45 {
|
||||||
t.Errorf("the mesh defines %d seats rather than 44; the set is closed, so a change here is "+
|
t.Errorf("the mesh defines %d seats rather than 45; the set is closed, so a change here is "+
|
||||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +1,66 @@
|
|||||||
package catalogue
|
package catalogue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/beside"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// boundKeys are every key the files of a module ask of ${bound:forge-account:<key>}, read from the module's own
|
||||||
|
// tree on the shelf, so the test offers what ssh-client asks however often it asks one more (issue 985).
|
||||||
|
func boundKeys(t *testing.T, module, provision string) map[string]bool {
|
||||||
|
t.Helper()
|
||||||
|
ask := regexp.MustCompile(`\$\{bound:` + regexp.QuoteMeta(provision) + `:([A-Za-z0-9_.-]+)\}`)
|
||||||
|
keys := map[string]bool{}
|
||||||
|
root := filepath.Join(beside.Catalogue(t), module)
|
||||||
|
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() || strings.HasSuffix(path, "_test.go") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, m := range ask.FindAllStringSubmatch(string(body), -1) {
|
||||||
|
keys[m[1]] = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading what %s asks of %s: %v", module, provision, err)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
// The ssh-client module, composed as a machine receives it (novox/hq to-be 29, research 027/03): a
|
// The ssh-client module, composed as a machine receives it (novox/hq to-be 29, research 027/03): a
|
||||||
// region at the START of the operator's ~/.ssh/config that includes ~/.ssh/config.d/* — first,
|
// region at the START of the operator's ~/.ssh/config that includes ~/.ssh/config.d/* — first,
|
||||||
// because ssh takes the first value it finds for each option — and the mesh's Host blocks as the
|
// because ssh takes the first value it finds for each option — and the mesh's Host blocks as the
|
||||||
// whole of ~/.ssh/config.d/00-mesh, every other node with its account. ~/.ssh and ~/.ssh/config.d
|
// whole of ~/.ssh/config.d/00-mesh, every other node with its account. ~/.ssh and ~/.ssh/config.d
|
||||||
// are created 0700 and owned by the account; the operator's own config below the region is kept.
|
// are created 0700 and owned by the account; the operator's own config below the region is kept.
|
||||||
func TestSSHClientOwnsTheOperatorsSSHConfig(t *testing.T) {
|
func TestSSHClientOwnsTheOperatorsSSHConfig(t *testing.T) {
|
||||||
shelf := shelf(catalogueManifest(t, "ssh-client"))
|
shelf := shelf(catalogueManifest(t, "ssh-client"), catalogueManifest(t, "gitea"))
|
||||||
|
// ssh-client requires forge-account since mesh-catalog #340 (novox/hq issue 955), answered by the forge's
|
||||||
|
// holder on another node; this test is about the operator's config, so the forge is offered from there
|
||||||
|
// (issue 985).
|
||||||
|
// Every key ssh-client's files ask is offered, read from them (boundKeys); a few have a realistic value.
|
||||||
|
serves := map[string]any{}
|
||||||
|
for key := range boundKeys(t, "ssh-client", "forge-account") {
|
||||||
|
serves[key] = "sample-" + key
|
||||||
|
}
|
||||||
|
for key, value := range map[string]any{"ssh_host": "marge.internal", "ssh_port": 222,
|
||||||
|
"known_host": "[marge.internal]:222 ssh-ed25519 AAAA", "ssh_base": "ssh://git@marge.internal:222/",
|
||||||
|
"https_base": "https://git.example.org/"} {
|
||||||
|
serves[key] = value
|
||||||
|
}
|
||||||
|
forge := World{Offered: map[string][]Provider{
|
||||||
|
"forge-account": {{Node: "marge", At: "marge.internal", Module: "gitea", Serves: serves}}}}
|
||||||
got, err := Resolve(shelf, []string{"ssh-client"},
|
got, err := Resolve(shelf, []string{"ssh-client"},
|
||||||
Node{Name: "homer", At: "homer.internal", Account: "jo", AccountHome: "/home/jo"}, World{})
|
Node{Name: "homer", At: "homer.internal", Account: "jo", AccountHome: "/home/jo"}, forge)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"}
|
|||||||
|
|
||||||
// MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this
|
// MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this
|
||||||
// package cannot import; a test keeps them agreeing.
|
// package cannot import; a test keeps them agreeing.
|
||||||
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED"}
|
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED", "CHECKS_READY", "CHECKS_OTHER"}
|
||||||
|
|
||||||
// StreamDeclaration is one stream a module owns.
|
// StreamDeclaration is one stream a module owns.
|
||||||
type StreamDeclaration struct {
|
type StreamDeclaration struct {
|
||||||
|
|||||||
@@ -160,7 +160,9 @@ var ControllerVerbs = []Verb{
|
|||||||
"the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " +
|
"the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " +
|
||||||
"again as its pull request is.",
|
"again as its pull request is.",
|
||||||
Input: schema(map[string]string{"group": "the group's id (none for one head checked again)",
|
Input: schema(map[string]string{"group": "the group's id (none for one head checked again)",
|
||||||
"members": "the members, as JSON, as delivery-order takes them"}, []string{"members"})},
|
"members": "the members, as JSON, as delivery-order takes them",
|
||||||
|
"ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"},
|
||||||
|
[]string{"members"})},
|
||||||
{Name: "deliver", Description: "The delivery's word that a walk waiting for it may start (novox/hq ADR 0239): its " +
|
{Name: "deliver", Description: "The delivery's word that a walk waiting for it may start (novox/hq ADR 0239): its " +
|
||||||
"first tier is asked at the controller's next pass. Refused for a walk that waits for nobody.",
|
"first tier is asked at the controller's next pass. Refused for a walk that waits for nobody.",
|
||||||
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
|
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
|
||||||
|
|||||||
@@ -112,6 +112,10 @@ type Machine struct {
|
|||||||
// NodeEngine and NodeTools are the builds it runs.
|
// NodeEngine and NodeTools are the builds it runs.
|
||||||
NodeEngine string `json:"node-engine,omitempty"`
|
NodeEngine string `json:"node-engine,omitempty"`
|
||||||
NodeTools string `json:"node-tools,omitempty"`
|
NodeTools string `json:"node-tools,omitempty"`
|
||||||
|
// Contract is the contract of the node-engine's newest health statement (0 when it never stated one): what
|
||||||
|
// says whether the machine takes handed files (novox/hq issue 969). A check that composed it without would
|
||||||
|
// judge a moved module the old way where the mesh hands it.
|
||||||
|
Contract int `json:"contract,omitempty"`
|
||||||
// Capabilities are what it reported it can do; the detail kept only where it is a version.
|
// Capabilities are what it reported it can do; the detail kept only where it is a version.
|
||||||
Capabilities []Capability `json:"capabilities,omitempty"`
|
Capabilities []Capability `json:"capabilities,omitempty"`
|
||||||
// Site, Hub, Public: where it is on the private network — its site (a pseudonym), whether it is the
|
// Site, Hub, Public: where it is on the private network — its site (a pseudonym), whether it is the
|
||||||
|
|||||||
@@ -119,6 +119,8 @@ type PlanBatch struct {
|
|||||||
ClosesAt time.Time `json:"closes_at"`
|
ClosesAt time.Time `json:"closes_at"`
|
||||||
AtMost time.Time `json:"at_most"`
|
AtMost time.Time `json:"at_most"`
|
||||||
Behind string `json:"behind,omitempty"`
|
Behind string `json:"behind,omitempty"`
|
||||||
|
// BehindWhy says why the batch waits behind that walk: what the two share (issue 973).
|
||||||
|
BehindWhy string `json:"behind_why,omitempty"`
|
||||||
// Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such
|
// Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such
|
||||||
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
|
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
|
||||||
// build, 2026-10-10).
|
// build, 2026-10-10).
|
||||||
|
|||||||
@@ -81,18 +81,10 @@ type ProvisionedCollection struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ProvisionedCredential is one provision the module requires, under one local name, and its provider (issue 760);
|
// ProvisionedCredential is one provision the module requires, under one local name, and its provider (issue 760);
|
||||||
// with a name, a credential issued for this holder alone (issue 757).
|
// with a name, a credential issued for this holder alone (issue 757). It is the composition's own type, not a copy:
|
||||||
type ProvisionedCredential struct {
|
// the plan is kept and read back as the one it was sent as, so no field of it (contributes, at, from, as, names) can
|
||||||
Name string `json:"name,omitempty"`
|
// be lost between the two (issue 923).
|
||||||
Provision string `json:"provision"`
|
type ProvisionedCredential = catalogue.PlanProvision
|
||||||
Local string `json:"local,omitempty"`
|
|
||||||
Provider string `json:"provider"`
|
|
||||||
ProviderModule string `json:"provider-module"`
|
|
||||||
// CredentialForm is the form the provider's offer declares for its credential (issue 768): shared, pair,
|
|
||||||
// issued or given; empty when the provider's manifest was not in hand.
|
|
||||||
CredentialForm string `json:"credential-form,omitempty"`
|
|
||||||
Path string `json:"path,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProvisionedSecret is one of the module's own secrets, by its name in the manifest. Path, when set, is the file
|
// ProvisionedSecret is one of the module's own secrets, by its name in the manifest. Path, when set, is the file
|
||||||
// holding the secret alone (the manifest's `own-secrets`), which the client fills with its value and nothing else.
|
// holding the secret alone (the manifest's `own-secrets`), which the client fills with its value and nothing else.
|
||||||
|
|||||||
@@ -352,3 +352,42 @@ func TestAPlanMayHoldSeveralNamedItemsOfOneCollectionAndRefusesABadOne(t *testin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A kept plan's provisions keep every field the composition sets** (issue 923): the names, the place, the machine
|
||||||
|
// and the identity the provider's helper writes into the consumer's entry, and what the manifest contributes, are
|
||||||
|
// read back as they were sent, so the assignments mirror carries them to the provider.
|
||||||
|
func TestAKeptPlansProvisionsKeepContributesAtFromAsAndNames(t *testing.T) {
|
||||||
|
sent := catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "route-proxy",
|
||||||
|
Contributes: map[string]any{"database": "gitea"}, At: "10.0.0.1", From: "novox", As: "gitea@novox",
|
||||||
|
Names: map[string]string{"name-web": "git.example.org", "internal-name": "gitea.mesh"}}
|
||||||
|
raw, err := json.Marshal(map[string]any{"gitea": map[string]any{
|
||||||
|
"provisions": []catalogue.PlanProvision{sent}, "required-by": "forge"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var kept map[string]keptPlan
|
||||||
|
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := kept["gitea"].Provisions
|
||||||
|
if len(got) != 1 || !reflect.DeepEqual(got[0], sent) {
|
||||||
|
t.Fatalf("the kept plan's provisions are %+v, want %+v", got, sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A kept plan's provision keeps the tool the vault asks for a software-issued credential (issue 928).
|
||||||
|
func TestAKeptPlansProvisionKeepsItsCredentialTool(t *testing.T) {
|
||||||
|
sent := catalogue.PlanProvision{Provision: "gitea-read", Provider: "novox", ProviderModule: "gitea",
|
||||||
|
CredentialForm: catalogue.CredentialFormIssued, CredentialTool: "gitea_agent_token"}
|
||||||
|
raw, err := json.Marshal(map[string]any{"claude-code": map[string]any{"provisions": []catalogue.PlanProvision{sent}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var kept map[string]keptPlan
|
||||||
|
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := kept["claude-code"].Provisions; len(got) != 1 || got[0].CredentialTool != "gitea_agent_token" {
|
||||||
|
t.Fatalf("the kept plan's provisions are %+v, want the tool kept", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -143,9 +143,9 @@ func MeshNamed(stream, name string) bool {
|
|||||||
return false // the mesh defines no durable consumer on a bucket's stream, or the memberships'
|
return false // the mesh defines no durable consumer on a bucket's stream, or the memberships'
|
||||||
case stream == "NODES":
|
case stream == "NODES":
|
||||||
return true
|
return true
|
||||||
case strings.HasPrefix(stream, "SEAT_"):
|
case strings.HasPrefix(stream, "SEAT_") || broker.IsCheckStream(stream):
|
||||||
return strings.HasSuffix(name, "_worker")
|
return strings.HasSuffix(name, "_worker")
|
||||||
case name == broker.ControllerName:
|
case name == broker.ControllerName || name == broker.MergesConsumerName:
|
||||||
return true
|
return true
|
||||||
case stream == broker.EventsStream:
|
case stream == broker.EventsStream:
|
||||||
return strings.Contains(name, "_")
|
return strings.Contains(name, "_")
|
||||||
@@ -156,10 +156,12 @@ func MeshNamed(stream, name string) bool {
|
|||||||
// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
|
// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
|
||||||
func ConsumerInWords(stream, name string) string {
|
func ConsumerInWords(stream, name string) string {
|
||||||
switch {
|
switch {
|
||||||
case name == broker.ControllerName:
|
case name == broker.ControllerName || name == broker.MergesConsumerName:
|
||||||
return "the controller's consumer on " + stream
|
return "the controller's consumer on " + stream
|
||||||
case stream == "NODES":
|
case stream == "NODES":
|
||||||
return "how " + name + " hears what it should be (its declaration consumer)"
|
return "how " + name + " hears what it should be (its declaration consumer)"
|
||||||
|
case broker.IsCheckStream(stream):
|
||||||
|
return "the worker every holder of " + broker.TheBuildSeat + " takes its checks from (" + stream + ")"
|
||||||
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
|
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
|
||||||
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
|
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
|
||||||
return "the worker every holder of " + seat + " takes its asks from"
|
return "the worker every holder of " + seat + " takes its asks from"
|
||||||
|
|||||||
@@ -144,6 +144,12 @@ type CheckRequest struct {
|
|||||||
Repo string `json:"repo"`
|
Repo string `json:"repo"`
|
||||||
Number int `json:"number,omitempty"`
|
Number int `json:"number,omitempty"`
|
||||||
Base string `json:"base,omitempty"`
|
Base string `json:"base,omitempty"`
|
||||||
|
// Trunk is the base branch's head the check is asked against, as the forge announced it with the head (novox/hq ADR
|
||||||
|
// 0406 §4): with the head, what the pull request's key says the current check is for, and what a waiting request
|
||||||
|
// is dropped by and a running check stopped by when the key moves.
|
||||||
|
Trunk string `json:"trunk,omitempty"`
|
||||||
|
// Ready says the pull request is marked ready (ADR 0406 §3): its check is queued before every other check.
|
||||||
|
Ready bool `json:"ready,omitempty"`
|
||||||
// Paths are the files the pull request changes, for the width of its rebuild.
|
// Paths are the files the pull request changes, for the width of its rebuild.
|
||||||
Paths []string `json:"paths,omitempty"`
|
Paths []string `json:"paths,omitempty"`
|
||||||
// Beside are the repositories the check reads next to this one, each cloned at the ref given — the
|
// Beside are the repositories the check reads next to this one, each cloned at the ref given — the
|
||||||
|
|||||||
+208
-74
@@ -76,11 +76,9 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
|
|||||||
}
|
}
|
||||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil {
|
||||||
return fmt.Errorf("cannot submit a build: %w", err)
|
return fmt.Errorf("cannot submit a build: %w", err)
|
||||||
}
|
}
|
||||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
|
||||||
signalAsked()
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,11 +110,9 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
|||||||
// be assumed, because nothing else will ever say so.
|
// be assumed, because nothing else will ever say so.
|
||||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil {
|
||||||
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
|
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
|
||||||
}
|
}
|
||||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
|
||||||
signalAsked()
|
|
||||||
|
|
||||||
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
||||||
defer cancelWait()
|
defer cancelWait()
|
||||||
@@ -162,10 +158,29 @@ type natsMachine struct {
|
|||||||
js *broker.JetStream
|
js *broker.JetStream
|
||||||
on string
|
on string
|
||||||
seat string
|
seat string
|
||||||
sub *nats.Subscription
|
// subs are the queues this machine takes from, in the order it takes from them: the seat's own, where a delivery
|
||||||
|
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
|
||||||
|
subs []*takeQueue
|
||||||
opts MachineOptions
|
opts MachineOptions
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// takeQueue is one queue a machine pulls from.
|
||||||
|
type takeQueue struct {
|
||||||
|
sub *nats.Subscription
|
||||||
|
// kind is what its asks are: a build, a ready check, or another check.
|
||||||
|
kind string
|
||||||
|
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
|
||||||
|
worker broker.Consumer
|
||||||
|
bound bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kinds of queue a machine takes from, in the order it takes from them.
|
||||||
|
const (
|
||||||
|
queueBuilds = "builds"
|
||||||
|
queueReady = "ready"
|
||||||
|
queueOther = "other"
|
||||||
|
)
|
||||||
|
|
||||||
// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219).
|
// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219).
|
||||||
type MachineOptions struct {
|
type MachineOptions struct {
|
||||||
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
||||||
@@ -207,47 +222,56 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (m *natsMachine) Close() {
|
func (m *natsMachine) Close() {
|
||||||
if m.sub != nil {
|
for _, q := range m.subs {
|
||||||
_ = m.sub.Unsubscribe()
|
if q.sub != nil {
|
||||||
|
_ = q.sub.Unsubscribe()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Take binds to the role's worker and pulls one request at a time, handing each over.
|
// Take binds to the role's workers and pulls one request at a time, handing each over.
|
||||||
//
|
//
|
||||||
// **Bound, never created.** The work queue and the worker on it are the controller's to define
|
// **Bound, never created.** The work queues and the workers on them are the controller's to define
|
||||||
// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said
|
// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said
|
||||||
// as the mesh's to answer rather than quietly created with whatever this client defaults to.
|
// as the mesh's to answer rather than quietly created with whatever this client defaults to.
|
||||||
//
|
//
|
||||||
// **Pulled, one at a time, by whichever holder is free** (novox/hq ADR 0190). Every machine holding
|
// **Pulled, one at a time, by whichever holder is free** (novox/hq ADR 0190). Every machine holding
|
||||||
// the role binds this same worker; a machine asks for the next request only when it has finished
|
// the role binds these same workers; a machine asks for the next request only when it has finished
|
||||||
// the last, so a slow machine never holds an ask an idle one could take, and a machine that took
|
// the last, so a slow machine never holds an ask an idle one could take, and a machine that took
|
||||||
// five at once would run five container builds against one runtime and finish all of them slower
|
// five at once would run five container builds against one runtime and finish all of them slower
|
||||||
// than the first.
|
// than the first.
|
||||||
|
//
|
||||||
|
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
|
||||||
|
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
|
||||||
|
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
|
||||||
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||||
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
||||||
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||||
if !found {
|
if !found {
|
||||||
return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat)
|
return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat)
|
||||||
}
|
}
|
||||||
|
m.subs = []*takeQueue{{kind: queueBuilds, worker: worker}}
|
||||||
// **The consumer's own filter, not the one subject this machine cares about.** The client checks
|
if m.seat == broker.TheBuildSeat {
|
||||||
// what is asked for against the consumer's filter and refuses anything that is not the same —
|
for i, w := range broker.CheckWorkers() {
|
||||||
// "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered
|
kind := queueReady
|
||||||
// on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different
|
if i == 1 {
|
||||||
// consumers, which is why it is written down here.
|
kind = queueOther
|
||||||
filter := worker.Filters[0]
|
}
|
||||||
sub, err := m.js.Context().PullSubscribe(filter, worker.Name,
|
m.subs = append(m.subs, &takeQueue{kind: kind, worker: w})
|
||||||
nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
}
|
||||||
if err != nil {
|
}
|
||||||
|
// The seat's own queue is bound at once and its absence is this machine's error, as it always was. The check queues
|
||||||
|
// are bound when they are there: a controller that has not made them yet is no reason to take no build.
|
||||||
|
if err := m.bind(m.subs[0]); err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"this machine cannot take work from %s: %w. The mesh creates that queue and this "+
|
"this machine cannot take work from %s: %w. The mesh creates that queue and this "+
|
||||||
"machine's worker on it, and a build machine may not create one itself — so this is "+
|
"machine's worker on it, and a build machine may not create one itself — so this is "+
|
||||||
"the mesh's to answer, not this machine's", m.seat, err)
|
"the mesh's to answer, not this machine's", m.seat, err)
|
||||||
}
|
}
|
||||||
m.sub = sub
|
|
||||||
if m.opts.Taking != nil {
|
if m.opts.Taking != nil {
|
||||||
m.opts.Taking()
|
m.opts.Taking()
|
||||||
}
|
}
|
||||||
|
var bindAgain time.Time
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
@@ -264,29 +288,31 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// One, and wait a while for it; an empty queue is a timeout, which is the normal state of a
|
if time.Now().After(bindAgain) {
|
||||||
// machine with nothing to build, and is asked again.
|
for _, q := range m.subs[1:] {
|
||||||
// Asked for a few seconds at a time when the holder can be paused, so a pause reaches a pull
|
if !q.bound {
|
||||||
// already waiting within that, rather than when the client's own wait runs out.
|
if err := m.bind(q); err != nil {
|
||||||
asking, endAsking := ctx, func() {}
|
fmt.Fprintf(os.Stderr, "the %s check queue is not there to take from yet: %v\n", q.kind, err)
|
||||||
if m.opts.Paused != nil {
|
}
|
||||||
asking, endAsking = context.WithTimeout(ctx, pausedFetch)
|
}
|
||||||
|
}
|
||||||
|
bindAgain = time.Now().Add(30 * time.Second)
|
||||||
}
|
}
|
||||||
fetched, err := sub.Fetch(1, nats.Context(asking))
|
msg, q, err := m.next(ctx)
|
||||||
endAsking()
|
|
||||||
switch {
|
switch {
|
||||||
case ctx.Err() != nil:
|
case ctx.Err() != nil:
|
||||||
// Ours ended: the machine is being stopped.
|
// Ours ended: the machine is being stopped.
|
||||||
return nil
|
return nil
|
||||||
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
|
case msg != nil:
|
||||||
|
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout), err == nil:
|
||||||
// **An empty queue, not the end.** A fetch on a context without a deadline waits the
|
// **An empty queue, not the end.** A fetch on a context without a deadline waits the
|
||||||
// client's own while and then says the deadline passed — the client's, not ours. Read
|
// client's own while and then says the deadline passed — the client's, not ours. Read
|
||||||
// as "stop", every idle build machine exited clean every half minute and was started
|
// as "stop", every idle build machine exited clean every half minute and was started
|
||||||
// again by its supervisor, which looked like a crash loop with nothing in the log to
|
// again by its supervisor, which looked like a crash loop with nothing in the log to
|
||||||
// say why (2026-10-03, the first build agents). Asked again.
|
// say why (2026-10-03, the first build agents). Asked again.
|
||||||
continue
|
continue
|
||||||
case err != nil:
|
default:
|
||||||
if sub.IsValid() {
|
if m.subs[0].sub.IsValid() {
|
||||||
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
||||||
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
||||||
time.Sleep(time.Second)
|
time.Sleep(time.Second)
|
||||||
@@ -294,49 +320,151 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
|||||||
}
|
}
|
||||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||||
}
|
}
|
||||||
for _, msg := range fetched {
|
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
if m.opts.Paused != nil && m.opts.Paused() {
|
||||||
if m.opts.Paused != nil && m.opts.Paused() {
|
_ = msg.Nak()
|
||||||
_ = msg.Nak()
|
continue
|
||||||
|
}
|
||||||
|
var request BuildRequest
|
||||||
|
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||||
|
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||||
|
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||||
|
_ = msg.Term()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||||
|
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||||
|
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||||
|
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||||
|
// exception, and a holder that refused every build while its grant was missing would
|
||||||
|
// stop the mesh building for it.
|
||||||
|
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||||
|
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||||
|
} else if cancelled {
|
||||||
|
endCancelled(ctx, build)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||||
|
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||||
|
if q.kind == queueOther {
|
||||||
|
if why := m.judgedAlready(ctx, request); why != "" {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||||
|
_ = msg.Ack()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var request BuildRequest
|
}
|
||||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
// landing then is.
|
||||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||||
_ = msg.Term()
|
_ = msg.Nak()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
// as the controller says it for its own long handlers, so the server neither hands
|
||||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
// the ask to a second machine nor counts the wait against its deliveries.
|
||||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
working := make(chan struct{})
|
||||||
// exception, and a holder that refused every build while its grant was missing would
|
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||||
// stop the mesh building for it.
|
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
// deleted when it ends, whatever the outcome.
|
||||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
do(ctx, build)
|
||||||
} else if cancelled {
|
stopRunning()
|
||||||
endCancelled(ctx, build)
|
close(working)
|
||||||
continue
|
}
|
||||||
}
|
}
|
||||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
|
||||||
// landing then is.
|
// bind binds a queue's worker, which exists and is not created here.
|
||||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
//
|
||||||
_ = msg.Nak()
|
// **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for
|
||||||
continue
|
// against the consumer's filter and refuses anything that is not the same — "subject does not match consumer" — so
|
||||||
}
|
// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
|
||||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
// twice now, on two different consumers, which is why it is written down here.
|
||||||
// as the controller says it for its own long handlers, so the server neither hands
|
func (m *natsMachine) bind(q *takeQueue) error {
|
||||||
// the ask to a second machine nor counts the wait against its deliveries.
|
sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
|
||||||
working := make(chan struct{})
|
nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
|
||||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
if err != nil {
|
||||||
do(ctx, build)
|
return err
|
||||||
close(working)
|
}
|
||||||
|
q.sub, q.bound = sub, true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
|
||||||
|
// on for a second. Nothing, with no error, when there was nothing.
|
||||||
|
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||||
|
var last error
|
||||||
|
for _, q := range m.subs {
|
||||||
|
if !q.bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||||
|
fetched, err := q.sub.Fetch(1, nats.Context(trying))
|
||||||
|
end()
|
||||||
|
if len(fetched) > 0 {
|
||||||
|
return fetched[0], q, nil
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) &&
|
||||||
|
!errors.Is(err, context.Canceled) {
|
||||||
|
last = err
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if last != nil {
|
||||||
|
return nil, nil, last
|
||||||
|
}
|
||||||
|
waiting, end := context.WithTimeout(ctx, takeIdleWait)
|
||||||
|
defer end()
|
||||||
|
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
|
||||||
|
if len(fetched) > 0 {
|
||||||
|
return fetched[0], m.subs[0], nil
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) &&
|
||||||
|
!errors.Is(err, context.Canceled) {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedAlready says why a check just taken from the other checks' queue need not run, or nothing: its pull request's key
|
||||||
|
// holds its head and trunk commit and a verdict. A key that cannot be read, or none, runs the check (the key is
|
||||||
|
// mesh-delivery's, and a check wrongly dropped is worse than one run twice).
|
||||||
|
func (m *natsMachine) judgedAlready(ctx context.Context, r BuildRequest) string {
|
||||||
|
c := r.Check
|
||||||
|
if c == nil || c.Try || (c.Group == "" && c.Number <= 0) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if c.Group == "" {
|
||||||
|
key, found, err := ReadPullKey(ctx, m.js.Conn(), c.Owner, c.Repo, c.Number)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read the key of %s/%s#%d, so its check runs: %v\n", c.Owner, c.Repo, c.Number, err)
|
||||||
|
} else if found && key.Judged(r.Ref, c.Trunk) {
|
||||||
|
return fmt.Sprintf("%s/%s#%d is judged already at %.12s on %.12s (%s)", c.Owner, c.Repo, c.Number, r.Ref, c.Trunk,
|
||||||
|
key.Verdict)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m.beingChecked(ctx, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// beingChecked says why a request need not run, or nothing: another builder's check of the same head and trunk commit is
|
||||||
|
// running, and says so under its pull request's key in the bucket of the checks running (ADR 0406 §3). A key a builder
|
||||||
|
// stopped renewing is gone within the bucket's age, and the request is taken then. Never the ask's own id: a redelivered ask
|
||||||
|
// is its own, and a key an older builder wrote that names no head or trunk is no match.
|
||||||
|
func (m *natsMachine) beingChecked(ctx context.Context, r BuildRequest) string {
|
||||||
|
running, ok, err := ReadRunning(ctx, m.js.Conn(), r)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read whether %s is being checked, so its check runs: %v\n", r.ID, err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if ok && running.Build != r.ID && running.Same(r) {
|
||||||
|
return fmt.Sprintf("%s is being checked at %.12s on %.12s by %s as %s", r.ID, r.Ref, r.Check.Trunk, running.Builder,
|
||||||
|
running.Build)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
type natsBuild struct {
|
type natsBuild struct {
|
||||||
@@ -398,6 +526,12 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
|||||||
|
|
||||||
func (b *natsBuild) Done() error { return b.msg.Ack() }
|
func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||||
|
|
||||||
|
// WatchPull has moved called once the pull request's key moves on from what this check was asked for; the stop ends the
|
||||||
|
// watch (see WatchPull). A request that is no pull request's check watches nothing.
|
||||||
|
func (b *natsBuild) WatchPull(ctx context.Context, moved func(PullKey)) (func(), error) {
|
||||||
|
return WatchPull(ctx, b.js.Conn(), b.request, moved)
|
||||||
|
}
|
||||||
|
|
||||||
// Began publishes that this machine has taken the build, into the stream like the outcome, so a
|
// Began publishes that this machine has taken the build, into the stream like the outcome, so a
|
||||||
// reader that asks afterwards sees when it started as well as how it ended.
|
// reader that asks afterwards sees when it started as well as how it ended.
|
||||||
func (b *natsBuild) Began(ctx context.Context) error {
|
func (b *natsBuild) Began(ctx context.Context) error {
|
||||||
|
|||||||
@@ -44,6 +44,12 @@ func aBusWithTheBuildRole(t *testing.T) *broker.JetStream {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// And the two check queues' workers, which the controller makes beside the streams (novox/hq ADR 0406 §3).
|
||||||
|
for _, w := range broker.CheckWorkers() {
|
||||||
|
if err := js.EnsureConsumer(w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
clean := func() {
|
clean := func() {
|
||||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||||
for _, s := range broker.MeshStreams() {
|
for _, s := range broker.MeshStreams() {
|
||||||
|
|||||||
@@ -179,6 +179,19 @@ func (b OverNATS) PublishMembership(ctx context.Context, node, module string, bo
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PublishMembershipAfter issues a membership only if the one read at sequence seq is still the newest of its
|
||||||
|
// subject: a push that issued another since is not overwritten, and the refusal comes back (novox/hq issue 930).
|
||||||
|
func (b OverNATS) PublishMembershipAfter(ctx context.Context, node, module string, body []byte, seq uint64) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, MembershipWait)
|
||||||
|
defer cancel()
|
||||||
|
_, err := b.JS.Publish(broker.MembershipSubject(node, module), body, nats.Context(ctx),
|
||||||
|
nats.ExpectLastSequencePerSubject(seq))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("issuing %s on %s its membership after sequence %d: %w", module, node, seq, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
|
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
|
||||||
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
|
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A pull request's check is queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
|
||||||
|
|
||||||
|
// SupersededByKey is the outcome of a check that stopped because its pull request's key moved on: its head or its trunk
|
||||||
|
// commit is no longer the one it was asked for. Said as the check's failure, as a kill is; nothing is asked of the forge.
|
||||||
|
const SupersededByKey = "superseded: the pull request's key moved on"
|
||||||
|
|
||||||
|
// unsafeToken is what may not stand in a subject token or a key.
|
||||||
|
var unsafeToken = strings.NewReplacer(".", "_", "*", "_", ">", "_", " ", "_", "/", "_", "\t", "_", "\n", "_")
|
||||||
|
|
||||||
|
// tokenOf is a name as one subject token: any dot, wildcard or space replaced by an underscore.
|
||||||
|
func tokenOf(name string) string {
|
||||||
|
if name == "" {
|
||||||
|
return "_"
|
||||||
|
}
|
||||||
|
return unsafeToken.Replace(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullKeyOf is a pull request's key in mesh-delivery's bucket, and the last tokens of the subject its check is asked on:
|
||||||
|
// `<owner>.<repo>.<number>`, a dot in a name made an underscore.
|
||||||
|
func PullKeyOf(owner, repo string, number int) string {
|
||||||
|
return tokenOf(owner) + "." + tokenOf(repo) + "." + strconv.Itoa(number)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckSubjectOf is the subject a check is asked on, and whether it has one of its own: a check of a pull request on
|
||||||
|
// the ready queue or the other, by the pull request; a delivery group's composed check by `group.<name>`; any other
|
||||||
|
// check (a branch tried before its pull request) by its own id, so it replaces nothing.
|
||||||
|
func CheckSubjectOf(seat string, r BuildRequest) (string, bool) {
|
||||||
|
c := r.Check
|
||||||
|
if seat != broker.TheBuildSeat || c == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
prefix := broker.CheckSubjectPrefix(seat, c.Ready)
|
||||||
|
switch {
|
||||||
|
case c.Group != "":
|
||||||
|
return prefix + "group." + tokenOf(c.Group), true
|
||||||
|
case c.Try || c.Number <= 0:
|
||||||
|
return prefix + "try." + tokenOf(r.ID), true
|
||||||
|
}
|
||||||
|
return prefix + PullKeyOf(c.Owner, c.Repo, c.Number), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// WorkSubjectOf is where an ask is put: a check on the build seat on its own queue, anything else on the seat's.
|
||||||
|
func WorkSubjectOf(seat string, r BuildRequest) string {
|
||||||
|
if subject, ok := CheckSubjectOf(seat, r); ok {
|
||||||
|
return subject
|
||||||
|
}
|
||||||
|
return BuildWorkOf(seat)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullKey is what mesh-delivery keeps of a pull request (ADR 0406 §4): its head, the trunk commit its current check is
|
||||||
|
// asked against, and its verdict once judged.
|
||||||
|
type PullKey struct {
|
||||||
|
Head string `json:"head"`
|
||||||
|
Trunk string `json:"trunk"`
|
||||||
|
Verdict string `json:"verdict,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// sameCommit says two spellings name one commit; either may be short, but not under twelve characters.
|
||||||
|
func sameCommit(a, b string) bool {
|
||||||
|
return a == b || SameHead(a, b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Moved says a key no longer holds what a check was asked for: its head or its trunk commit differs.
|
||||||
|
func (k PullKey) Moved(head, trunk string) bool {
|
||||||
|
return !sameCommit(k.Head, head) || (trunk != "" && k.Trunk != "" && !sameCommit(k.Trunk, trunk))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Judged says a key holds the verdict of the check of this head against this trunk commit.
|
||||||
|
func (k PullKey) Judged(head, trunk string) bool {
|
||||||
|
// A verdict of a check that ran: a pass, a warning or a fail. An error is a check that could not run, and a request
|
||||||
|
// after it must run (issue 974, review 2).
|
||||||
|
judgedBy := k.Verdict == "pass" || k.Verdict == "warning" || k.Verdict == "fail"
|
||||||
|
return judgedBy && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadPullKey reads a pull request's key from mesh-delivery's bucket: false when it has none.
|
||||||
|
func ReadPullKey(ctx context.Context, conn *nats.Conn, owner, repo string, number int) (PullKey, bool, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return PullKey{}, false, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.PullsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return PullKey{}, false, fmt.Errorf("the pull requests' keys: %w", err)
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, PullKeyOf(owner, repo, number))
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return PullKey{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return PullKey{}, false, err
|
||||||
|
}
|
||||||
|
var key PullKey
|
||||||
|
if err := json.Unmarshal(e.Value(), &key); err != nil {
|
||||||
|
return PullKey{}, false, fmt.Errorf("the key of %s/%s#%d is not a pull request's: %w", owner, repo, number, err)
|
||||||
|
}
|
||||||
|
return key, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WatchPull calls moved once when the pull request's key moves on from what this check was asked for — its head or its
|
||||||
|
// trunk commit differs — and not for a key that only gains its verdict. It hears updates, not the key as it stands: the
|
||||||
|
// key and the request are written for one announcement by two writers, so a key found older than the request is not
|
||||||
|
// a move. The check is then stopped by whoever asked for it, without asking the forge (novox/hq issue 726). The stop
|
||||||
|
// ends the watch.
|
||||||
|
func WatchPull(ctx context.Context, conn *nats.Conn, r BuildRequest, moved func(PullKey)) (stop func(), err error) {
|
||||||
|
c := r.Check
|
||||||
|
if c == nil || c.Group != "" || c.Try || c.Number <= 0 {
|
||||||
|
return func() {}, nil
|
||||||
|
}
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.PullsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the pull requests' keys: %w", err)
|
||||||
|
}
|
||||||
|
watching, end := context.WithCancel(ctx)
|
||||||
|
w, err := kv.Watch(watching, PullKeyOf(c.Owner, c.Repo, c.Number), jetstream.UpdatesOnly())
|
||||||
|
if err != nil {
|
||||||
|
end()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer w.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-watching.Done():
|
||||||
|
return
|
||||||
|
case e, ok := <-w.Updates():
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var key PullKey
|
||||||
|
if json.Unmarshal(e.Value(), &key) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if key.Moved(r.Ref, c.Trunk) {
|
||||||
|
moved(key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return end, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
|
||||||
|
// waits on the first when all three are empty.
|
||||||
|
const (
|
||||||
|
checkTakeWait = 30 * time.Millisecond
|
||||||
|
takeIdleWait = time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
|
||||||
|
// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
|
||||||
|
type CheckRunning struct {
|
||||||
|
Head string `json:"head"`
|
||||||
|
Trunk string `json:"trunk"`
|
||||||
|
Builder string `json:"builder"`
|
||||||
|
Build string `json:"build"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
|
||||||
|
// composed check; none for a try, which no other request replaces.
|
||||||
|
func RunningKeyOf(r BuildRequest) (string, bool) {
|
||||||
|
c := r.Check
|
||||||
|
switch {
|
||||||
|
case c == nil, c.Try:
|
||||||
|
return "", false
|
||||||
|
case c.Group != "":
|
||||||
|
return "group." + tokenOf(c.Group), true
|
||||||
|
case c.Number <= 0:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return PullKeyOf(c.Owner, c.Repo, c.Number), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// runningBucket opens the bucket of the checks running.
|
||||||
|
func runningBucket(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.CheckRunningBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the checks running: %w", err)
|
||||||
|
}
|
||||||
|
return kv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadRunning reads the key of the check running for a request's pull request: false when there is none.
|
||||||
|
func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRunning, bool, error) {
|
||||||
|
key, ok := RunningKeyOf(r)
|
||||||
|
if !ok {
|
||||||
|
return CheckRunning{}, false, nil
|
||||||
|
}
|
||||||
|
kv, err := runningBucket(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return CheckRunning{}, false, err
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return CheckRunning{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return CheckRunning{}, false, err
|
||||||
|
}
|
||||||
|
var running CheckRunning
|
||||||
|
if err := json.Unmarshal(e.Value(), &running); err != nil {
|
||||||
|
return CheckRunning{}, false, nil // a key of a builder from before this means no match
|
||||||
|
}
|
||||||
|
return running, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same says a running check is for this request's head and trunk commit: both said, and the same. A key that names neither
|
||||||
|
// (an older builder's) is no match.
|
||||||
|
func (c CheckRunning) Same(r BuildRequest) bool {
|
||||||
|
return r.Check != nil && c.Head != "" && c.Trunk != "" && r.Check.Trunk != "" && sameCommit(c.Head, r.Ref) &&
|
||||||
|
sameCommit(c.Trunk, r.Check.Trunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepRunning says a check is running: its key put now, put again at each heartbeat, and deleted by the stop it returns,
|
||||||
|
// whatever the outcome. Said to nobody that cannot be told: a bucket that is not there is no reason not to run the check.
|
||||||
|
func keepRunning(ctx context.Context, conn *nats.Conn, r BuildRequest, builder string, every time.Duration) (stop func()) {
|
||||||
|
key, ok := RunningKeyOf(r)
|
||||||
|
if !ok {
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
kv, err := runningBucket(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s runs without saying so: %v\n", r.ID, err)
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
trunk := ""
|
||||||
|
if r.Check != nil {
|
||||||
|
trunk = r.Check.Trunk
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(CheckRunning{Head: r.Ref, Trunk: trunk, Builder: builder, Build: r.ID})
|
||||||
|
put := func() {
|
||||||
|
putting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||||
|
defer end()
|
||||||
|
if _, err := kv.Put(putting, key, body); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s could not say it is running: %v\n", r.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put()
|
||||||
|
done := make(chan struct{})
|
||||||
|
finished := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(finished)
|
||||||
|
tick := time.NewTicker(every)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
put()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return func() {
|
||||||
|
close(done)
|
||||||
|
<-finished
|
||||||
|
deleting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||||
|
defer end()
|
||||||
|
if err := kv.Delete(deleting, key); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s could not take its running key away; it ages out: %v\n", r.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,650 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The checks of pull requests are queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
|
||||||
|
|
||||||
|
func aPullCheck(id string, number int, head, trunk string, ready bool) BuildRequest {
|
||||||
|
return BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
||||||
|
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: number, Trunk: trunk, Ready: ready}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The subject a check is asked on says its queue and its pull request: ready or other, `<owner>.<repo>.<number>` with
|
||||||
|
// a dot in a name made an underscore, `group.<name>` for a group's, the id for a try; a build is on the seat's own queue.
|
||||||
|
func TestACheckIsAskedOnItsQueueBySubject(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
r BuildRequest
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{aPullCheck("b1", 7, "h", "t", false), "mesh.seat.node-build-agent.check.other.novox.a.7"},
|
||||||
|
{aPullCheck("b2", 7, "h", "t", true), "mesh.seat.node-build-agent.check.ready.novox.a.7"},
|
||||||
|
{BuildRequest{ID: "b3", Check: &CheckRequest{Owner: "no.vox", Repo: "a.b", Number: 12}}, "mesh.seat.node-build-agent.check.other.no_vox.a_b.12"},
|
||||||
|
{BuildRequest{ID: "b4", Check: &CheckRequest{Owner: "novox", Repo: "a", Group: "g.1", Ready: true}}, "mesh.seat.node-build-agent.check.ready.group.g_1"},
|
||||||
|
{BuildRequest{ID: "b5", Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}}, "mesh.seat.node-build-agent.check.other.try.b5"},
|
||||||
|
{BuildRequest{ID: "b6", Repository: "/r"}, "mesh.seat.node-build-agent.accept.build"},
|
||||||
|
{BuildRequest{ID: "b7", Repository: "/r", DryRun: true}, "mesh.seat.node-build-agent.accept.build"},
|
||||||
|
} {
|
||||||
|
if got := WorkSubjectOf(TheBuildMachine, c.r); got != c.want {
|
||||||
|
t.Errorf("%s is asked on %s; want %s", c.r.ID, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The retired role has no check queues: its checks wait with its builds.
|
||||||
|
if got := WorkSubjectOf(TheBuildMachineBefore, aPullCheck("b8", 7, "h", "t", true)); got != "mesh.seat.mesh-build-machine.accept.build" {
|
||||||
|
t.Errorf("a check of the retired role is asked on %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func queueOf(t *testing.T, js *broker.JetStream) Queue {
|
||||||
|
t.Helper()
|
||||||
|
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
// A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is
|
||||||
|
// left with one queued check (acceptance 5).
|
||||||
|
func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
for _, r := range []BuildRequest{
|
||||||
|
aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
|
||||||
|
aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
|
||||||
|
aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
|
||||||
|
aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
|
||||||
|
aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
|
||||||
|
} {
|
||||||
|
if err := asks.Ask(ctx, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var got []string
|
||||||
|
for _, a := range queueOf(t, js).Asks {
|
||||||
|
if a.State != AskWaiting {
|
||||||
|
t.Errorf("%s is %s", a.ID, a.State)
|
||||||
|
}
|
||||||
|
got = append(got, a.ID)
|
||||||
|
}
|
||||||
|
if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the
|
||||||
|
// order they were asked.
|
||||||
|
func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
for _, r := range []BuildRequest{
|
||||||
|
aPullCheck("other-1", 1, "h1", "t", false),
|
||||||
|
aPullCheck("other-2", 2, "h2", "t", false),
|
||||||
|
aPullCheck("ready-3", 3, "h3", "t", true),
|
||||||
|
{ID: "delivery-4", Repository: "/d"},
|
||||||
|
aPullCheck("other-5", 5, "h5", "t", false),
|
||||||
|
aPullCheck("ready-6", 6, "h6", "t", true),
|
||||||
|
} {
|
||||||
|
if err := asks.Ask(ctx, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
took := make(chan string, 8)
|
||||||
|
machine := MachineOverNATS(js, "anchor")
|
||||||
|
defer machine.Close()
|
||||||
|
go func() {
|
||||||
|
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
took <- work.Request().ID
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
var order []string
|
||||||
|
for len(order) < 6 {
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
order = append(order, id)
|
||||||
|
case <-time.After(15 * time.Second):
|
||||||
|
t.Fatalf("taken so far: %v", order)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
|
||||||
|
t.Fatalf("taken as %v; want %v", order, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module.
|
||||||
|
func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
|
||||||
|
t.Helper()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.PullsBucket})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return kv
|
||||||
|
}
|
||||||
|
|
||||||
|
func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) {
|
||||||
|
t.Helper()
|
||||||
|
body, _ := json.Marshal(k)
|
||||||
|
if _, err := kv.Put(t.Context(), PullKeyOf("novox", "a", number), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a
|
||||||
|
// builder takes it; one for a head the key does not say judged is run.
|
||||||
|
func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
kv := aKeyBucket(t, js)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
|
||||||
|
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged
|
||||||
|
putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
for _, r := range []BuildRequest{
|
||||||
|
aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
|
||||||
|
aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
|
||||||
|
aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
|
||||||
|
} {
|
||||||
|
if err := asks.Ask(ctx, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
took := make(chan string, 8)
|
||||||
|
machine := MachineOverNATS(js, "anchor")
|
||||||
|
defer machine.Close()
|
||||||
|
go func() {
|
||||||
|
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
took <- work.Request().ID
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
var ran []string
|
||||||
|
for len(ran) < 2 {
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
ran = append(ran, id)
|
||||||
|
case <-time.After(15 * time.Second):
|
||||||
|
t.Fatalf("run so far: %v", ran)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
|
||||||
|
t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
if n := len(queueOf(t, js).Asks); n != 0 {
|
||||||
|
t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a
|
||||||
|
// group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all.
|
||||||
|
func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
kv := aKeyBucket(t, js)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
moved := make(chan PullKey, 4)
|
||||||
|
req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
|
||||||
|
end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer end()
|
||||||
|
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
|
||||||
|
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
|
||||||
|
select {
|
||||||
|
case k := <-moved:
|
||||||
|
t.Fatalf("stopped by a key that moved nothing: %+v", k)
|
||||||
|
case <-time.After(500 * time.Millisecond):
|
||||||
|
}
|
||||||
|
putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
|
||||||
|
select {
|
||||||
|
case k := <-moved:
|
||||||
|
if k.Head != "head-7-111111111" {
|
||||||
|
t.Errorf("stopped by %+v", k)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("a key whose head moved did not stop the check")
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the trunk commit.
|
||||||
|
req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
|
||||||
|
end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer end2()
|
||||||
|
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
|
||||||
|
select {
|
||||||
|
case <-moved:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("a key whose trunk commit moved did not stop the check")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
|
||||||
|
// have is the pull request's own.
|
||||||
|
group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
|
||||||
|
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
|
||||||
|
end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer end3()
|
||||||
|
putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
|
||||||
|
select {
|
||||||
|
case k := <-moved:
|
||||||
|
t.Fatalf("a group's check was stopped by %+v", k)
|
||||||
|
case <-time.After(500 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) {
|
||||||
|
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"}
|
||||||
|
if k.Moved("aaaaaaaaaaaa", "tttttttttttt") || !k.Moved("bbbbbbbbbbbbbbbb", "tttttttttttt") || !k.Moved("aaaaaaaaaaaa", "uuuuuuuuuuuuuu") {
|
||||||
|
t.Error("moved is wrong")
|
||||||
|
}
|
||||||
|
if !k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111") || k.Judged("aaaaaaaaaaaa1111", "") || (PullKey{Head: "a", Trunk: "t"}).Judged("a", "t") {
|
||||||
|
t.Error("judged is wrong")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an
|
||||||
|
// ask is removed from that stream.
|
||||||
|
func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
for _, r := range []BuildRequest{
|
||||||
|
aPullCheck("other-1", 1, "h1", "t", false),
|
||||||
|
aPullCheck("ready-2", 2, "h2", "t", true),
|
||||||
|
{ID: "delivery-3", Repository: "/d"},
|
||||||
|
} {
|
||||||
|
if err := asks.Ask(ctx, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
q := queueOf(t, js)
|
||||||
|
var ids, streams []string
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
ids, streams = append(ids, a.ID), append(streams, a.Stream)
|
||||||
|
}
|
||||||
|
if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
|
||||||
|
t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
|
||||||
|
}
|
||||||
|
if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
|
||||||
|
t.Fatalf("streams %v; want %v", streams, want)
|
||||||
|
}
|
||||||
|
ready, _ := q.Find("ready-2")
|
||||||
|
if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
|
||||||
|
t.Errorf("the ready check is handed out by %s", w.Name)
|
||||||
|
}
|
||||||
|
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
|
||||||
|
t.Fatalf("deleted %d, %v", removed, err)
|
||||||
|
}
|
||||||
|
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
|
||||||
|
t.Fatalf("deleted again: %d, %v", removed, err)
|
||||||
|
}
|
||||||
|
if n := len(queueOf(t, js).Asks); n != 2 {
|
||||||
|
t.Errorf("%d asks left; want 2", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test
|
||||||
|
// (the mesh's is two minutes): a key not renewed within it is gone.
|
||||||
|
func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetstream.KeyValue {
|
||||||
|
t.Helper()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: age})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return kv
|
||||||
|
}
|
||||||
|
|
||||||
|
// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it:
|
||||||
|
// the bucket of running checks names the head and the trunk, the builder and the build (issue 974).
|
||||||
|
func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
aKeyBucket(t, js)
|
||||||
|
aRunningBucket(t, js, 2*time.Minute)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
running := make(chan string, 4)
|
||||||
|
release := make(chan struct{})
|
||||||
|
a := MachineOverNATS(js, "builder-a")
|
||||||
|
defer a.Close()
|
||||||
|
go func() {
|
||||||
|
_ = a.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
running <- work.Request().ID
|
||||||
|
<-release
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-running:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("builder a took nothing")
|
||||||
|
}
|
||||||
|
// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b := MachineOverNATS(js, "builder-b")
|
||||||
|
defer b.Close()
|
||||||
|
took := make(chan string, 4)
|
||||||
|
go func() {
|
||||||
|
_ = b.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
took <- work.Request().ID
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
t.Fatalf("builder b took %s, which builder a is running a check of", id)
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
}
|
||||||
|
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
|
||||||
|
t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
|
||||||
|
}
|
||||||
|
// Another trunk commit is another check.
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
if id != "moved" {
|
||||||
|
t.Fatalf("builder b took %s", id)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("a check against another trunk commit was dropped too")
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died
|
||||||
|
// running a check leaves nothing that holds the next one back for longer than the bucket's age.
|
||||||
|
func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
aKeyBucket(t, js)
|
||||||
|
kv := aRunningBucket(t, js, 2*time.Second)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
|
||||||
|
if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
took := make(chan string, 4)
|
||||||
|
b := MachineOverNATS(js, "builder-b")
|
||||||
|
defer b.Close()
|
||||||
|
go func() {
|
||||||
|
_ = b.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
took <- work.Request().ID
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
t.Fatalf("%s was taken while the key was fresh", id)
|
||||||
|
case <-time.After(1200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
time.Sleep(2500 * time.Millisecond) // the key was not renewed
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
if id != "after-it-aged" {
|
||||||
|
t.Fatalf("took %s", id)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("the request was not taken after the key aged out")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check
|
||||||
|
// ends, whatever the outcome.
|
||||||
|
func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
kv := aRunningBucket(t, js, 2*time.Second)
|
||||||
|
was := heartbeatWhileWorking
|
||||||
|
heartbeatWhileWorking = 400 * time.Millisecond
|
||||||
|
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
running := make(chan struct{}, 1)
|
||||||
|
release := make(chan struct{})
|
||||||
|
m := MachineOverNATS(js, "builder-a")
|
||||||
|
defer m.Close()
|
||||||
|
go func() {
|
||||||
|
_ = m.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
running <- struct{}{}
|
||||||
|
<-release
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
<-running
|
||||||
|
key := PullKeyOf("novox", "a", 7)
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("no key while the check runs: %v", err)
|
||||||
|
}
|
||||||
|
var got CheckRunning
|
||||||
|
_ = json.Unmarshal(e.Value(), &got)
|
||||||
|
if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
|
||||||
|
t.Errorf("the key says %+v", got)
|
||||||
|
}
|
||||||
|
time.Sleep(3 * time.Second) // longer than the bucket's age
|
||||||
|
if _, err := kv.Get(ctx, key); err != nil {
|
||||||
|
t.Fatalf("the key aged out under a check still running: %v", err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||||
|
if _, err := kv.Get(ctx, key); err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(end) {
|
||||||
|
t.Fatal("the key stays after the check ended")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and
|
||||||
|
// trunk commit must run (issue 974, review 2).
|
||||||
|
func TestAnErrorVerdictJudgesNothing(t *testing.T) {
|
||||||
|
for verdict, judged := range map[string]bool{"pass": true, "warning": true, "fail": true, "error": false, "": false, "pending": false} {
|
||||||
|
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: verdict}
|
||||||
|
if got := k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111"); got != judged {
|
||||||
|
t.Errorf("a key with verdict %q judged = %v; want %v", verdict, got, judged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and
|
||||||
|
// run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it.
|
||||||
|
func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
kv := aKeyBucket(t, js)
|
||||||
|
aRunningBucket(t, js, 2*time.Minute)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
asks := BuildsOn(js, TheBuildMachine)
|
||||||
|
outcomes := make(chan BuildResult, 8)
|
||||||
|
watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
|
||||||
|
var r BuildResult
|
||||||
|
if json.Unmarshal(m.Data, &r) == nil {
|
||||||
|
outcomes <- r
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = watching.Unsubscribe() }()
|
||||||
|
|
||||||
|
ran := make(chan string, 4)
|
||||||
|
m := MachineOverNATS(js, "builder-a")
|
||||||
|
defer m.Close()
|
||||||
|
go func() {
|
||||||
|
_ = m.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
building, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
ran <- work.Request().ID
|
||||||
|
stopped := make(chan struct{}, 1)
|
||||||
|
end, err := work.(interface {
|
||||||
|
WatchPull(context.Context, func(PullKey)) (func(), error)
|
||||||
|
}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer end()
|
||||||
|
if work.Request().ID == "older" {
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
|
||||||
|
case <-time.After(15 * time.Second):
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
|
||||||
|
}
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if id := <-ran; id != "older" {
|
||||||
|
t.Fatalf("took %s", id)
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond) // the watch is made
|
||||||
|
// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
|
||||||
|
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
|
||||||
|
if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case id := <-ran:
|
||||||
|
if id != "newer" {
|
||||||
|
t.Fatalf("took %s", id)
|
||||||
|
}
|
||||||
|
case <-time.After(15 * time.Second):
|
||||||
|
t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
|
||||||
|
}
|
||||||
|
got := map[string]string{}
|
||||||
|
for len(got) < 2 {
|
||||||
|
select {
|
||||||
|
case r := <-outcomes:
|
||||||
|
got[r.ID] = r.Failed
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatalf("outcomes so far: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got["older"] != SupersededByKey || got["newer"] != "" {
|
||||||
|
t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again
|
||||||
|
// when the worker's wait is up.
|
||||||
|
func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
|
||||||
|
for _, ready := range []bool{false, true} {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
name, worker := "other", broker.ChecksOtherWorker
|
||||||
|
if ready {
|
||||||
|
name, worker = "ready", broker.ChecksReadyWorker
|
||||||
|
}
|
||||||
|
// A worker whose wait is a second, as the mesh's is a minute.
|
||||||
|
for _, w := range broker.CheckWorkers() {
|
||||||
|
if w.Name == worker {
|
||||||
|
w.AckWaitSeconds = 1
|
||||||
|
if err := js.EnsureConsumer(w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The first builder takes it and dies: its pull is fetched, never acknowledged.
|
||||||
|
if _, err := fetchOne(js, worker); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
took := make(chan BuildRequest, 2)
|
||||||
|
second := MachineOverNATS(js, "builder-b")
|
||||||
|
go func() {
|
||||||
|
_ = second.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
took <- work.Request()
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case r := <-took:
|
||||||
|
if r.ID != "dies-"+name {
|
||||||
|
t.Errorf("%s: took %s", name, r.ID)
|
||||||
|
}
|
||||||
|
case <-time.After(15 * time.Second):
|
||||||
|
t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
|
||||||
|
}
|
||||||
|
second.Close()
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would.
|
||||||
|
func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) {
|
||||||
|
for _, w := range broker.CheckWorkers() {
|
||||||
|
if w.Name != worker {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sub, err := js.Context().PullSubscribe(w.Filters[0], w.Name, nats.Bind(w.Stream, w.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return got[0], nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
@@ -76,6 +76,10 @@ const (
|
|||||||
// KeyChecked: a pull request's merge check was judged (novox/hq to-be 45 §9) — the verdict, its
|
// KeyChecked: a pull request's merge check was judged (novox/hq to-be 45 §9) — the verdict, its
|
||||||
// summary and its report, for the forge's holder to set as the pull request's status.
|
// summary and its report, for the forge's holder to set as the pull request's status.
|
||||||
KeyChecked = "checked"
|
KeyChecked = "checked"
|
||||||
|
// KeySent: a plan's tier was sent to a machine, and the send kept (novox/hq ADR 0406 §5, issue 974): the plan, the tier,
|
||||||
|
// the machine, the modules and when. What an outsider hears of a rollout; the controller's own next step does not
|
||||||
|
// wait for it.
|
||||||
|
KeySent = "sent"
|
||||||
// KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines —
|
// KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines —
|
||||||
// was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its
|
// was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its
|
||||||
// steps from. Said after every save of the serving controller; a delivery's holder also asks for the
|
// steps from. Said after every save of the serving controller; a delivery's holder also asks for the
|
||||||
@@ -260,6 +264,9 @@ type PullUpdated struct {
|
|||||||
// check's second layer; MergeCheckSaid says the announcer looked.
|
// check's second layer; MergeCheckSaid says the announcer looked.
|
||||||
MergeCheck bool `json:"merge_check,omitempty"`
|
MergeCheck bool `json:"merge_check,omitempty"`
|
||||||
MergeCheckSaid bool `json:"merge_check_said,omitempty"`
|
MergeCheckSaid bool `json:"merge_check_said,omitempty"`
|
||||||
|
// Ready says the pull request is marked ready (novox/hq ADR 0406 §3): not the forge's word, mesh-delivery's, said
|
||||||
|
// to the controller with the head to check again; its check is queued before every other check.
|
||||||
|
Ready bool `json:"ready,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Checked is a pull request's merge check, judged: what the controller says as `checked`.
|
// Checked is a pull request's merge check, judged: what the controller says as `checked`.
|
||||||
|
|||||||
+99
-49
@@ -62,11 +62,12 @@ type QueuedAsk struct {
|
|||||||
Path string `json:"path,omitempty"`
|
Path string `json:"path,omitempty"`
|
||||||
Ref string `json:"ref,omitempty"`
|
Ref string `json:"ref,omitempty"`
|
||||||
AskedAt time.Time `json:"asked-at,omitempty"`
|
AskedAt time.Time `json:"asked-at,omitempty"`
|
||||||
// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
|
// Stored is when the stream took it.
|
||||||
Stored time.Time `json:"stored,omitempty"`
|
Stored time.Time `json:"stored,omitempty"`
|
||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
|
// Stream is the work queue holding it: the seat's own, or one of the two check queues (novox/hq ADR 0406 §3). Its
|
||||||
movedFrom string
|
// Seq is the sequence there.
|
||||||
|
Stream string `json:"stream,omitempty"`
|
||||||
// On and Started are the holder building an in-flight ask and when it started, from its started
|
// On and Started are the holder building an in-flight ask and when it started, from its started
|
||||||
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
|
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
|
||||||
// back, to be handed out again — with Started its start there.
|
// back, to be handed out again — with Started its start there.
|
||||||
@@ -201,62 +202,113 @@ func startedAt(s BuildStart) time.Time {
|
|||||||
return t
|
return t
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the
|
// queueStream is one stream a seat's asks wait in, and the worker that hands them out.
|
||||||
// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO,
|
type queueStream struct {
|
||||||
// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches.
|
Worker broker.Consumer
|
||||||
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
|
Filter string
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueStreamsOf are the streams a seat's asks wait in, in the order a builder takes from them (novox/hq ADR 0406 §3): the
|
||||||
|
// seat's own queue, where a delivery build waits, then the checks of pull requests marked ready, then every other check.
|
||||||
|
// Only the current build role has the check queues; the retired one's checks wait on its own.
|
||||||
|
func queueStreamsOf(seat string) []queueStream {
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||||
|
streams := []queueStream{{Worker: worker, Filter: BuildWorkOf(seat)}}
|
||||||
|
if seat == broker.TheBuildSeat {
|
||||||
|
for _, w := range broker.CheckWorkers() {
|
||||||
|
streams = append(streams, queueStream{Worker: w, Filter: w.Filters[0]})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return streams
|
||||||
|
}
|
||||||
|
|
||||||
|
// WorkerOf is the worker that hands out an ask, by the queue it waits in.
|
||||||
|
func WorkerOf(seat string, ask QueuedAsk) broker.Consumer {
|
||||||
|
streams := queueStreamsOf(seat)
|
||||||
|
for _, qs := range streams {
|
||||||
|
if qs.Worker.Stream == ask.Stream {
|
||||||
|
return qs.Worker
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return streams[0].Worker
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadQueue reads a build seat's work queues: the streams' asks, the workers' positions, and what the seat's events
|
||||||
|
// said about the builds behind them. Through the JetStream API alone (STREAM.INFO, CONSUMER.INFO, STREAM.MSG.GET),
|
||||||
|
// which only the controller's account reaches. The asks come in the order they are handed out: the seat's own queue's,
|
||||||
|
// the ready checks', the other checks'.
|
||||||
|
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
|
||||||
q := Queue{Seat: seat}
|
q := Queue{Seat: seat}
|
||||||
api, err := jetstream.New(js.Conn())
|
api, err := jetstream.New(js.Conn())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return q, err
|
return q, err
|
||||||
}
|
}
|
||||||
stream, err := api.Stream(ctx, worker.Stream)
|
type read struct {
|
||||||
if err != nil {
|
asks []QueuedAsk
|
||||||
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
|
delivered uint64
|
||||||
|
ackPending int
|
||||||
}
|
}
|
||||||
consumer, err := stream.Consumer(ctx, worker.Name)
|
var reads []read
|
||||||
switch {
|
for i, qs := range queueStreamsOf(seat) {
|
||||||
case errors.Is(err, jetstream.ErrConsumerNotFound):
|
stream, err := api.Stream(ctx, qs.Worker.Stream)
|
||||||
// No holder has ever been given a worker: everything in the stream is waiting.
|
|
||||||
case err != nil:
|
|
||||||
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
|
||||||
default:
|
|
||||||
info, err := consumer.Info(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, qs.Worker.Stream, err)
|
||||||
|
}
|
||||||
|
var r read
|
||||||
|
consumer, err := stream.Consumer(ctx, qs.Worker.Name)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, jetstream.ErrConsumerNotFound):
|
||||||
|
// No holder has ever been given a worker: everything in the stream is waiting.
|
||||||
|
case err != nil:
|
||||||
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||||
|
default:
|
||||||
|
info, err := consumer.Info(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||||
|
}
|
||||||
|
r.delivered, r.ackPending = info.Delivered.Stream, info.NumAckPending
|
||||||
|
if i == 0 {
|
||||||
|
q.Delivered, q.MaxDeliver = info.Delivered.Stream, info.Config.MaxDeliver
|
||||||
|
}
|
||||||
|
q.AckPending += info.NumAckPending
|
||||||
|
if q.MaxDeliver == 0 {
|
||||||
|
q.MaxDeliver = info.Config.MaxDeliver
|
||||||
|
}
|
||||||
}
|
}
|
||||||
q.Delivered = info.Delivered.Stream
|
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
|
||||||
q.AckPending = info.NumAckPending
|
// is a handful of reads, never the history.
|
||||||
q.MaxDeliver = info.Config.MaxDeliver
|
var seq uint64 = 1
|
||||||
|
for n := 0; n < 10000; n++ {
|
||||||
|
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(qs.Filter))
|
||||||
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
||||||
|
}
|
||||||
|
a := askOf(msg)
|
||||||
|
a.Stream = qs.Worker.Stream
|
||||||
|
r.asks = append(r.asks, a)
|
||||||
|
seq = msg.Sequence + 1
|
||||||
|
}
|
||||||
|
reads = append(reads, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
|
// What the events say, from shortly before the oldest ask behind a worker: its start, if any, came after it was asked.
|
||||||
// is a handful of reads, never the history.
|
|
||||||
var asks []QueuedAsk
|
|
||||||
var seq uint64 = 1
|
|
||||||
for n := 0; n < 10000; n++ {
|
|
||||||
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
|
|
||||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
|
||||||
}
|
|
||||||
asks = append(asks, askOf(msg))
|
|
||||||
seq = msg.Sequence + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// What the events say, from shortly before the oldest ask behind the worker: its start, if any,
|
|
||||||
// came after it was asked.
|
|
||||||
var since time.Time
|
var since time.Time
|
||||||
for _, a := range asks {
|
behind := false
|
||||||
if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) {
|
for _, r := range reads {
|
||||||
since = a.AskedAt
|
for _, a := range r.asks {
|
||||||
|
if a.Seq <= r.delivered {
|
||||||
|
behind = true
|
||||||
|
if since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since)) {
|
||||||
|
since = a.AskedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
|
heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
|
||||||
if len(asks) > 0 && q.Delivered > 0 {
|
if behind {
|
||||||
if since.IsZero() {
|
if since.IsZero() {
|
||||||
since = time.Now().Add(-7 * 24 * time.Hour)
|
since = time.Now().Add(-7 * 24 * time.Hour)
|
||||||
}
|
}
|
||||||
@@ -264,17 +316,15 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
|||||||
return q, err
|
return q, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard)
|
for _, r := range reads {
|
||||||
|
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
|
||||||
|
}
|
||||||
return q, nil
|
return q, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
|
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
|
||||||
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
|
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
|
||||||
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
|
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
|
||||||
// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
|
|
||||||
if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
|
|
||||||
ask.Stored = at
|
|
||||||
}
|
|
||||||
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
||||||
r := ask.Request
|
r := ask.Request
|
||||||
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
||||||
@@ -282,7 +332,7 @@ func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
|
|||||||
ask.AskedAt = at
|
ask.AskedAt = at
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
|
// Nothing read is nothing kept: a half-read request would pass for a delivery.
|
||||||
ask.Request = BuildRequest{}
|
ask.Request = BuildRequest{}
|
||||||
}
|
}
|
||||||
return ask
|
return ask
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The kinds of ask, as the queue says them. **The kind decides the queue** (novox/hq ADR 0406 §3): a delivery build is on
|
||||||
|
// the seat's own queue, taken before any check; a check of a pull request marked ready is on the second; every other
|
||||||
|
// check on the third. Nothing orders the asks any more: a builder takes from the queues in that order, and the check
|
||||||
|
// queues keep one request a pull request. The reordering of one queue by moving asks to its tail, with its pause of the
|
||||||
|
// worker, its look every two seconds and its second copies, is gone (issue 558 is answered by the streams).
|
||||||
|
const (
|
||||||
|
KindDelivery = "delivery"
|
||||||
|
KindCheck = "check"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
|
||||||
|
// a dry run, or an ask nothing could be read from; a delivery for any other build.
|
||||||
|
func KindOf(r BuildRequest) string {
|
||||||
|
if r.ID == "" || r.Check != nil || r.DryRun {
|
||||||
|
return KindCheck
|
||||||
|
}
|
||||||
|
return KindDelivery
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteAsk removes a waiting or dead ask from the queue it is in, at its sequence. Says the sequence it removed, 0 when
|
||||||
|
// it was not there any more: taken and settled, or replaced by a newer request for its pull request.
|
||||||
|
func DeleteAsk(_ context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
|
||||||
|
stream := ask.Stream
|
||||||
|
if stream == "" {
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||||
|
stream = worker.Stream
|
||||||
|
}
|
||||||
|
err := js.Context().DeleteMsg(stream, ask.Seq)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return ask.Seq, nil
|
||||||
|
case errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
|
||||||
|
strings.Contains(err.Error(), "no message found"):
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The order the build queue hands asks out in (novox/hq issue 558).
|
|
||||||
//
|
|
||||||
// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served,
|
|
||||||
// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after
|
|
||||||
// it, waited on the build. So:
|
|
||||||
//
|
|
||||||
// - a delivery is handed out before any check;
|
|
||||||
// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
|
|
||||||
// waits for ever behind a stream of deliveries;
|
|
||||||
// - within a kind, first come, first served, by when it was asked;
|
|
||||||
// - a branch tried before its pull request goes after everything else, however long it has waited: it is
|
|
||||||
// never promoted ahead of a delivery, and never ahead of a pull request's check (novox/hq issue 686);
|
|
||||||
// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
|
|
||||||
// taken late breaks nothing.
|
|
||||||
//
|
|
||||||
// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out
|
|
||||||
// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller
|
|
||||||
// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where
|
|
||||||
// it was and published again, its body unchanged, under the same id — while the worker is paused, so no
|
|
||||||
// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched.
|
|
||||||
|
|
||||||
// The kinds of ask, as the queue says them.
|
|
||||||
const (
|
|
||||||
KindDelivery = "delivery"
|
|
||||||
KindCheck = "check"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it.
|
|
||||||
var CheckWaitsAtMost = 10 * time.Minute
|
|
||||||
|
|
||||||
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
|
|
||||||
// a dry run, or an ask nothing could be read from; a delivery for any other build.
|
|
||||||
func KindOf(r BuildRequest) string {
|
|
||||||
if r.ID == "" || r.Check != nil || r.DryRun {
|
|
||||||
return KindCheck
|
|
||||||
}
|
|
||||||
return KindDelivery
|
|
||||||
}
|
|
||||||
|
|
||||||
// askedOrStored is when an ask was asked, as its id says, or else when the queue took it.
|
|
||||||
func (a QueuedAsk) askedOrStored() time.Time {
|
|
||||||
if !a.AskedAt.IsZero() {
|
|
||||||
return a.AskedAt
|
|
||||||
}
|
|
||||||
return a.Stored
|
|
||||||
}
|
|
||||||
|
|
||||||
// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
|
|
||||||
// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
|
|
||||||
func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
|
|
||||||
var first, then, tries []QueuedAsk
|
|
||||||
for _, a := range waiting {
|
|
||||||
if IsTry(a.Request) {
|
|
||||||
tries = append(tries, a)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
|
|
||||||
then = append(then, a)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
first = append(first, a)
|
|
||||||
}
|
|
||||||
byAsking := func(s []QueuedAsk) {
|
|
||||||
sort.SliceStable(s, func(i, j int) bool {
|
|
||||||
if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) {
|
|
||||||
return ti.Before(tj)
|
|
||||||
}
|
|
||||||
return s[i].Seq < s[j].Seq
|
|
||||||
})
|
|
||||||
}
|
|
||||||
byAsking(first)
|
|
||||||
byAsking(then)
|
|
||||||
byAsking(tries)
|
|
||||||
return append(append(first, then...), tries...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
|
|
||||||
// after the longest start of order that the queue already holds in that order.
|
|
||||||
func toMove(queue, order []QueuedAsk) []QueuedAsk {
|
|
||||||
at := make(map[uint64]int, len(queue))
|
|
||||||
for i, a := range queue {
|
|
||||||
at[a.Seq] = i
|
|
||||||
}
|
|
||||||
kept, last := 0, -1
|
|
||||||
for kept < len(order) {
|
|
||||||
i, ok := at[order[kept].Seq]
|
|
||||||
if !ok || i < last {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
last = i
|
|
||||||
kept++
|
|
||||||
}
|
|
||||||
return order[kept:]
|
|
||||||
}
|
|
||||||
|
|
||||||
// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so
|
|
||||||
// there is one; the lock holds a test's two at once to that too.
|
|
||||||
var orderPasses sync.Mutex
|
|
||||||
|
|
||||||
// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the
|
|
||||||
// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving
|
|
||||||
// with at least this left, and the next pass moves the rest, so no move runs past the pause.
|
|
||||||
const (
|
|
||||||
pausedFor = 20 * time.Second
|
|
||||||
moveMargin = 5 * time.Second
|
|
||||||
publishWaits = 3 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in
|
|
||||||
// time, its id unread or not.
|
|
||||||
const QueuedAtHeader = "Mesh-Queued-At"
|
|
||||||
|
|
||||||
// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader,
|
|
||||||
// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose
|
|
||||||
// headers are its original's — and so never name that original's own sequence.
|
|
||||||
const MovedFromHeader = "Mesh-Moved-From"
|
|
||||||
|
|
||||||
// Pass is what one pass did: the asks it moved, and each second copy it removed, in words.
|
|
||||||
type Pass struct {
|
|
||||||
Moved []string
|
|
||||||
Removed []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in
|
|
||||||
// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving.
|
|
||||||
var asked = make(chan struct{}, 1)
|
|
||||||
|
|
||||||
// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass.
|
|
||||||
func signalAsked() {
|
|
||||||
select {
|
|
||||||
case asked <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue
|
|
||||||
// already in order is read and left alone; the worker is paused only while asks move. Run by the acting
|
|
||||||
// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused.
|
|
||||||
func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) {
|
|
||||||
var pass Pass
|
|
||||||
orderPasses.Lock()
|
|
||||||
defer orderPasses.Unlock()
|
|
||||||
api, err := jetstream.New(js.Conn())
|
|
||||||
if err != nil {
|
|
||||||
return pass, err
|
|
||||||
}
|
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
|
||||||
waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat)
|
|
||||||
if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) {
|
|
||||||
return pass, err
|
|
||||||
}
|
|
||||||
// Paused, then read again: what the worker handed out up to the pause is not waiting any more.
|
|
||||||
pauseEnds := time.Now().Add(pausedFor)
|
|
||||||
if delivered >= 0 {
|
|
||||||
if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil {
|
|
||||||
return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+
|
|
||||||
"first come, first served: %w", seat, err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil {
|
|
||||||
fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+
|
|
||||||
"itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil {
|
|
||||||
return pass, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stream, err := api.Stream(ctx, worker.Stream)
|
|
||||||
if err != nil {
|
|
||||||
return pass, err
|
|
||||||
}
|
|
||||||
// A second copy of one ask — a move that published and stopped before it deleted — goes first.
|
|
||||||
for _, a := range extra {
|
|
||||||
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
||||||
return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err)
|
|
||||||
}
|
|
||||||
pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s",
|
|
||||||
a.ID, a.Seq, a.movedFrom))
|
|
||||||
}
|
|
||||||
for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) {
|
|
||||||
if delivered >= 0 && time.Until(pauseEnds) < moveMargin {
|
|
||||||
// The rest waits for the next pass, under a pause of its own.
|
|
||||||
break
|
|
||||||
}
|
|
||||||
ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a)
|
|
||||||
if err != nil {
|
|
||||||
return pass, err
|
|
||||||
}
|
|
||||||
if ok {
|
|
||||||
pass.Moved = append(pass.Moved, a.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return pass, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one
|
|
||||||
// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no
|
|
||||||
// worker, so nothing is handed out to pause.
|
|
||||||
//
|
|
||||||
// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an
|
|
||||||
// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's
|
|
||||||
// first-queued time — a move that published its copy and stopped before it deleted the original. The
|
|
||||||
// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other
|
|
||||||
// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a
|
|
||||||
// real ask and stays.
|
|
||||||
func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) (
|
|
||||||
waiting, extra []QueuedAsk, delivered int64, err error) {
|
|
||||||
stream, err := api.Stream(ctx, worker.Stream)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
|
|
||||||
}
|
|
||||||
delivered = -1
|
|
||||||
if consumer, err := stream.Consumer(ctx, worker.Name); err == nil {
|
|
||||||
info, err := consumer.Info(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
|
||||||
}
|
|
||||||
delivered = int64(info.Delivered.Stream)
|
|
||||||
} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
|
|
||||||
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
|
||||||
}
|
|
||||||
// From the first: the stream holds only what is unsettled, so this is a handful of reads.
|
|
||||||
// Every message by sequence, with its id and first-queued time, to prove a copy against.
|
|
||||||
type held struct {
|
|
||||||
id string
|
|
||||||
stored time.Time
|
|
||||||
}
|
|
||||||
bySeq := map[string]held{}
|
|
||||||
var seq uint64 = 1
|
|
||||||
for n := 0; n < 10000; n++ {
|
|
||||||
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
|
|
||||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
|
||||||
}
|
|
||||||
seq = msg.Sequence + 1
|
|
||||||
a := askOf(msg)
|
|
||||||
a.movedFrom = msg.Header.Get(MovedFromHeader)
|
|
||||||
from, known := bySeq[a.movedFrom]
|
|
||||||
again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored)
|
|
||||||
bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored}
|
|
||||||
switch {
|
|
||||||
case int64(a.Seq) <= delivered:
|
|
||||||
case again:
|
|
||||||
extra = append(extra, a)
|
|
||||||
default:
|
|
||||||
waiting = append(waiting, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return waiting, extra, delivered, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The
|
|
||||||
// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two
|
|
||||||
// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask.
|
|
||||||
// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False
|
|
||||||
// when it was not moved.
|
|
||||||
func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string,
|
|
||||||
a QueuedAsk) (bool, error) {
|
|
||||||
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
|
|
||||||
info, err := consumer.Info(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if info.Delivered.Stream >= a.Seq {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
msg, err := stream.GetMsg(ctx, a.Seq)
|
|
||||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}}
|
|
||||||
for k, v := range msg.Header {
|
|
||||||
// The server's own headers stay behind: a message id would be taken for a duplicate and dropped.
|
|
||||||
if !strings.HasPrefix(k, "Nats-") {
|
|
||||||
again.Header[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Its place in time goes with it: when the stream first took it, as askOf read it; and the message it
|
|
||||||
// is a copy of, which proves it one.
|
|
||||||
again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano))
|
|
||||||
again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10))
|
|
||||||
// Inside the margin, so the publish ends while the worker is still paused.
|
|
||||||
publishing, cancel := context.WithTimeout(ctx, publishWaits)
|
|
||||||
ack, err := js.Context().PublishMsg(again, nats.Context(publishing))
|
|
||||||
cancel()
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err)
|
|
||||||
}
|
|
||||||
// **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the
|
|
||||||
// copy goes instead, so it is built once.
|
|
||||||
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
|
|
||||||
info, err := consumer.Info(ctx)
|
|
||||||
if err != nil || info.Delivered.Stream >= a.Seq {
|
|
||||||
if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) {
|
|
||||||
return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+
|
|
||||||
"removed; the next pass removes it: %w", a.ID, ack.Sequence, derr)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+
|
|
||||||
"removed and it stays where it was: %w", a.ID, err)
|
|
||||||
}
|
|
||||||
if info.Delivered.Stream >= ack.Sequence {
|
|
||||||
return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID)
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
||||||
return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+
|
|
||||||
"removes the second copy: %w", a.ID, a.Seq, err)
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this
|
|
||||||
// process says so, when the queue's last sequence moved — an ask from another process — and every
|
|
||||||
// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting
|
|
||||||
// whether this controller acts (Holding); said what a pass did.
|
|
||||||
func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool,
|
|
||||||
said func(format string, args ...any)) {
|
|
||||||
tick := time.NewTicker(orderLooks)
|
|
||||||
defer tick.Stop()
|
|
||||||
var lastSeq uint64
|
|
||||||
var lastPass time.Time
|
|
||||||
for {
|
|
||||||
signalled := false
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-asked:
|
|
||||||
signalled = true
|
|
||||||
case <-tick.C:
|
|
||||||
}
|
|
||||||
if !acting() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
name := seat()
|
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}})
|
|
||||||
info, err := js.Context().StreamInfo(worker.Stream)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
lastSeq, lastPass = info.State.LastSeq, time.Now()
|
|
||||||
passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second)
|
|
||||||
pass, err := PutInOrder(passing, js, name)
|
|
||||||
cancel()
|
|
||||||
for _, r := range pass.Removed {
|
|
||||||
said("the %s queue: removed %s", name, r)
|
|
||||||
}
|
|
||||||
if len(pass.Moved) > 0 {
|
|
||||||
said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", "))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
said("the %s queue could not be put in order: %v", name, err)
|
|
||||||
}
|
|
||||||
if info, err := js.Context().StreamInfo(worker.Stream); err == nil {
|
|
||||||
// Its own moves are not an ask.
|
|
||||||
lastSeq = info.State.LastSeq
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often
|
|
||||||
// it passes with nothing asked.
|
|
||||||
var (
|
|
||||||
orderLooks = 2 * time.Second
|
|
||||||
passEvery = 30 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it
|
|
||||||
// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more.
|
|
||||||
func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
|
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
|
||||||
gone := func(err error) bool {
|
|
||||||
return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
|
|
||||||
(err != nil && strings.Contains(err.Error(), "no message found"))
|
|
||||||
}
|
|
||||||
err := js.Context().DeleteMsg(worker.Stream, ask.Seq)
|
|
||||||
if err == nil {
|
|
||||||
return ask.Seq, nil
|
|
||||||
}
|
|
||||||
if !gone(err) {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if ask.ID == "" {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
q, err := ReadQueue(ctx, js, seat)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
for _, a := range q.Asks {
|
|
||||||
if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight {
|
|
||||||
if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) {
|
|
||||||
return 0, err
|
|
||||||
} else if err == nil {
|
|
||||||
return a.Seq, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
@@ -1,441 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
|
|
||||||
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
|
|
||||||
// first come, first served within a kind.
|
|
||||||
|
|
||||||
func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
|
|
||||||
r := BuildRequest{ID: id, Repository: "/r"}
|
|
||||||
if check {
|
|
||||||
r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
|
|
||||||
}
|
|
||||||
return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
|
|
||||||
}
|
|
||||||
|
|
||||||
func idsOf(asks []QueuedAsk) []string {
|
|
||||||
out := []string{}
|
|
||||||
for _, a := range asks {
|
|
||||||
out = append(out, a.ID)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
|
||||||
waiting := []QueuedAsk{
|
|
||||||
anAsk(1, "check-a", now.Add(-3*time.Minute), true),
|
|
||||||
anAsk(2, "check-b", now.Add(-2*time.Minute), true),
|
|
||||||
anAsk(3, "delivery-d", now.Add(-time.Minute), false),
|
|
||||||
}
|
|
||||||
want := []string{"delivery-d", "check-a", "check-b"}
|
|
||||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("handed out as %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
// Only the checks ahead of it move behind it; the delivery stays where it is.
|
|
||||||
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
|
|
||||||
t.Fatalf("moved %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
|
||||||
// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
|
|
||||||
waiting := []QueuedAsk{
|
|
||||||
anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
|
|
||||||
anAsk(6, "delivery-later", now.Add(-time.Minute), false),
|
|
||||||
anAsk(7, "check-old", now.Add(-11*time.Minute), true),
|
|
||||||
anAsk(8, "check-young", now.Add(-9*time.Minute), true),
|
|
||||||
}
|
|
||||||
want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
|
|
||||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("handed out as %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
|
|
||||||
t.Fatalf("moved %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
|
|
||||||
for _, r := range []BuildRequest{
|
|
||||||
{}, // unreadable: nothing was read from it
|
|
||||||
{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
|
|
||||||
{ID: "build-2", DryRun: true}, // looked at, never delivered
|
|
||||||
} {
|
|
||||||
if got := KindOf(r); got != KindCheck {
|
|
||||||
t.Errorf("%+v is a %s", r, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
|
|
||||||
t.Errorf("a build is a %s", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
|
|
||||||
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx, stop := context.WithCancel(context.Background())
|
|
||||||
defer stop()
|
|
||||||
asks := BuildsOn(js, TheBuildMachine)
|
|
||||||
now := time.Now()
|
|
||||||
for i, r := range []BuildRequest{
|
|
||||||
{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
|
|
||||||
{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
|
|
||||||
{ID: NewBuildID(now), Repository: "/d"},
|
|
||||||
} {
|
|
||||||
if i == 2 {
|
|
||||||
// And one nobody can read, whose kind cannot be told: it waits with the checks.
|
|
||||||
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := asks.Ask(ctx, r); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The acting controller's loop puts it in order, told by the asks.
|
|
||||||
go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
|
|
||||||
func(string, ...any) {})
|
|
||||||
time.Sleep(500 * time.Millisecond)
|
|
||||||
took := make(chan string, 4)
|
|
||||||
machine := MachineOverNATS(js, "anchor")
|
|
||||||
defer machine.Close()
|
|
||||||
go func() {
|
|
||||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
|
||||||
took <- work.Request().Repository
|
|
||||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
|
||||||
_ = work.Done()
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
var order []string
|
|
||||||
for len(order) < 3 {
|
|
||||||
select {
|
|
||||||
case r := <-took:
|
|
||||||
order = append(order, r)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatalf("taken so far: %v", order)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
|
|
||||||
t.Fatalf("taken as %v, want %v", order, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- the moves, against a real server ------------------------------------------------------------
|
|
||||||
|
|
||||||
func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
|
|
||||||
t.Helper()
|
|
||||||
body, _ := json.Marshal(r)
|
|
||||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func aCheck(asked time.Time, repository string) BuildRequest {
|
|
||||||
return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func aDelivery(asked time.Time, repository string) BuildRequest {
|
|
||||||
return BuildRequest{ID: NewBuildID(asked), Repository: repository}
|
|
||||||
}
|
|
||||||
|
|
||||||
// queued is the stream's asks, in queue order.
|
|
||||||
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
|
|
||||||
t.Helper()
|
|
||||||
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return q.Asks
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeAll runs a holder that records how often it was handed each repository.
|
|
||||||
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
|
|
||||||
t.Helper()
|
|
||||||
took := make(chan string, 100)
|
|
||||||
var mu sync.Mutex
|
|
||||||
counts := map[string]int{}
|
|
||||||
machine := MachineOverNATS(js, "anchor")
|
|
||||||
t.Cleanup(machine.Close)
|
|
||||||
go func() {
|
|
||||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
|
||||||
mu.Lock()
|
|
||||||
counts[work.Request().Repository]++
|
|
||||||
mu.Unlock()
|
|
||||||
took <- work.Request().Repository
|
|
||||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
|
||||||
_ = work.Done()
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
return took, func() map[string]int {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
out := map[string]int{}
|
|
||||||
for k, v := range counts {
|
|
||||||
out[k] = v
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
|
|
||||||
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
|
|
||||||
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
now := time.Now()
|
|
||||||
check := aCheck(now.Add(-2*time.Minute), "/c")
|
|
||||||
publishAsk(t, js, check)
|
|
||||||
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
|
|
||||||
moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
|
|
||||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
|
|
||||||
t.Fatalf("removed %v: %v", pass.Removed, err)
|
|
||||||
}
|
|
||||||
var got []string
|
|
||||||
for _, a := range queued(t, js) {
|
|
||||||
got = append(got, a.Repository)
|
|
||||||
}
|
|
||||||
if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("after the pass the queue holds %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And one a holder took before the copy was published: the copy goes, the holder's stays.
|
|
||||||
taken := aCheck(now, "/taken")
|
|
||||||
publishAsk(t, js, taken)
|
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
|
||||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
// Take everything waiting up to and including /taken, and keep /taken unsettled.
|
|
||||||
for {
|
|
||||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
|
||||||
if err != nil || len(msgs) == 0 {
|
|
||||||
t.Fatalf("nothing to take: %v", err)
|
|
||||||
}
|
|
||||||
var r BuildRequest
|
|
||||||
_ = json.Unmarshal(msgs[0].Data, &r)
|
|
||||||
if r.Repository == "/taken" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
_ = msgs[0].Ack()
|
|
||||||
}
|
|
||||||
moveCopyOf(t, js, taken.ID)
|
|
||||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
n := 0
|
|
||||||
for _, a := range queued(t, js) {
|
|
||||||
if a.ID == taken.ID {
|
|
||||||
n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n != 1 {
|
|
||||||
t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
|
|
||||||
// would leave it.
|
|
||||||
func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
|
|
||||||
t.Helper()
|
|
||||||
for _, a := range queued(t, js) {
|
|
||||||
if a.ID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(a.Request)
|
|
||||||
msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
|
|
||||||
msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
|
|
||||||
msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
|
|
||||||
if _, err := js.Context().PublishMsg(msg); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Fatalf("%s is not queued", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
|
|
||||||
// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
|
|
||||||
// copy, whose headers the delivery carries.
|
|
||||||
func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
ask := aCheck(time.Now(), "/again")
|
|
||||||
publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
|
|
||||||
publishAsk(t, js, ask)
|
|
||||||
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
|
||||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
|
||||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
var original *nats.Msg
|
|
||||||
for original == nil {
|
|
||||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
|
||||||
if err != nil || len(msgs) == 0 {
|
|
||||||
t.Fatalf("nothing to take: %v", err)
|
|
||||||
}
|
|
||||||
var r BuildRequest
|
|
||||||
_ = json.Unmarshal(msgs[0].Data, &r)
|
|
||||||
if r.ID == ask.ID {
|
|
||||||
original = msgs[0] // given up on, never settled: dead, and kept
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_ = msgs[0].Ack()
|
|
||||||
}
|
|
||||||
// Delivered again as DeliverAgain does: its headers, and its own.
|
|
||||||
again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
|
|
||||||
for k, v := range original.Header {
|
|
||||||
again.Header[k] = v
|
|
||||||
}
|
|
||||||
again.Header.Set(AgainHeader, "7")
|
|
||||||
if _, err := js.Context().PublishMsg(again); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
pass, err := PutInOrder(ctx, js, TheBuildMachine)
|
|
||||||
if err != nil || len(pass.Removed) != 0 {
|
|
||||||
t.Fatalf("removed %v: %v", pass.Removed, err)
|
|
||||||
}
|
|
||||||
n := 0
|
|
||||||
for _, a := range queued(t, js) {
|
|
||||||
if a.ID == ask.ID {
|
|
||||||
n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n != 2 {
|
|
||||||
t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Two passes at once: nothing is built twice, and nothing is lost.
|
|
||||||
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
defer stop()
|
|
||||||
took, counts := takeAll(t, ctx, js)
|
|
||||||
now := time.Now()
|
|
||||||
const asks = 24
|
|
||||||
var passes sync.WaitGroup
|
|
||||||
for p := 0; p < 2; p++ {
|
|
||||||
passes.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer passes.Done()
|
|
||||||
for i := 0; i < 15; i++ {
|
|
||||||
_, _ = PutInOrder(ctx, js, TheBuildMachine)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
for i := 0; i < asks; i++ {
|
|
||||||
repository := fmt.Sprintf("/%02d", i)
|
|
||||||
if i%3 == 2 {
|
|
||||||
publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
|
|
||||||
} else {
|
|
||||||
publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
passes.Wait()
|
|
||||||
for got := 0; got < asks; got++ {
|
|
||||||
select {
|
|
||||||
case <-took:
|
|
||||||
case <-time.After(15 * time.Second):
|
|
||||||
t.Fatalf("taken %d of %d: %v", got, asks, counts())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
c := counts()
|
|
||||||
for i := 0; i < asks; i++ {
|
|
||||||
if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
|
|
||||||
t.Errorf("/%02d was built %d times", i, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
|
|
||||||
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
defer stop()
|
|
||||||
now := time.Now()
|
|
||||||
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
|
|
||||||
publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
|
|
||||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
took, _ := takeAll(t, ctx, js)
|
|
||||||
var order []string
|
|
||||||
for len(order) < 2 {
|
|
||||||
select {
|
|
||||||
case r := <-took:
|
|
||||||
order = append(order, r)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatalf("taken so far: %v", order)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
|
|
||||||
t.Fatalf("taken as %v, want %v", order, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
|
|
||||||
// there was nothing left to delete.
|
|
||||||
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
now := time.Now()
|
|
||||||
publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
|
|
||||||
read := queued(t, js)[0]
|
|
||||||
publishAsk(t, js, aDelivery(now, "/d"))
|
|
||||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
|
||||||
t.Fatalf("moved %v: %v", pass.Moved, err)
|
|
||||||
}
|
|
||||||
removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
|
|
||||||
if err != nil || removed == 0 || removed == read.Seq {
|
|
||||||
t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
|
|
||||||
}
|
|
||||||
for _, a := range queued(t, js) {
|
|
||||||
if a.ID == read.ID {
|
|
||||||
t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
|
|
||||||
t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An ask nothing can be read from keeps its place in time when it moves.
|
|
||||||
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
|
|
||||||
js := aBusWithTheBuildRole(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first := queued(t, js)[0].Stored
|
|
||||||
time.Sleep(50 * time.Millisecond)
|
|
||||||
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
|
||||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
|
||||||
t.Fatalf("moved %v: %v", pass.Moved, err)
|
|
||||||
}
|
|
||||||
asks := queued(t, js)
|
|
||||||
if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
|
|
||||||
t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -68,6 +68,7 @@ type fakeControl struct {
|
|||||||
on *fakeInbound
|
on *fakeInbound
|
||||||
redelivered bool
|
redelivered bool
|
||||||
since time.Time
|
since time.Time
|
||||||
|
holdFails error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *fakeControl) Kind() string { return m.kind }
|
func (m *fakeControl) Kind() string { return m.kind }
|
||||||
@@ -88,6 +89,9 @@ func (m *fakeControl) HeldFor() time.Duration {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *fakeControl) Hold(time.Duration) error {
|
func (m *fakeControl) Hold(time.Duration) error {
|
||||||
|
if m.holdFails != nil {
|
||||||
|
return m.holdFails
|
||||||
|
}
|
||||||
if m.since.IsZero() {
|
if m.since.IsZero() {
|
||||||
m.since = time.Now()
|
m.since = time.Now()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -148,6 +148,24 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
defer func() { _ = followed.Unsubscribe() }()
|
defer func() { _ = followed.Unsubscribe() }()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// And the forge's merges, off a consumer of their own: no max-deliver and no reset, so a merge held for the
|
||||||
|
// plans waits for them however long (novox/hq ADR 0407 decision 3). Bound to the same channel: one loop.
|
||||||
|
if n.follows[KindSourceMoved] {
|
||||||
|
if events == nil {
|
||||||
|
events = make(chan *nats.Msg, Prefetch)
|
||||||
|
}
|
||||||
|
merges, err := standingBy(ctx, log.Default(), "EVENTS", func() (*nats.Subscription, error) {
|
||||||
|
return js.ChanSubscribe(MergedSubject, events, nats.Bind("EVENTS", broker.MergesConsumerName))
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to the forge's merges: %w", err)
|
||||||
|
}
|
||||||
|
if merges == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer func() { _ = merges.Unsubscribe() }()
|
||||||
|
}
|
||||||
|
|
||||||
// A connection that dropped is said, not discovered. A controller whose bus connection is gone
|
// A connection that dropped is said, not discovered. A controller whose bus connection is gone
|
||||||
// is a mesh where nothing can be told anything.
|
// is a mesh where nothing can be told anything.
|
||||||
gone := make(chan error, 1)
|
gone := make(chan error, 1)
|
||||||
|
|||||||
@@ -590,3 +590,77 @@ func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) {
|
|||||||
type sameAgain struct{}
|
type sameAgain struct{}
|
||||||
|
|
||||||
func (sameAgain) Heard(context.Context, Report) (bool, error) { return false, nil }
|
func (sameAgain) Heard(context.Context, Report) (bool, error) { return false, nil }
|
||||||
|
|
||||||
|
// manyTimesBusy is an upgrader whose plans are busy for its first merges, as another controller's hold is.
|
||||||
|
type manyTimesBusy struct {
|
||||||
|
toldAbout
|
||||||
|
busy int
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *manyTimesBusy) SourceMoved(context.Context, SourceMoved) error {
|
||||||
|
u.mu.Lock()
|
||||||
|
defer u.mu.Unlock()
|
||||||
|
u.calls++
|
||||||
|
if u.calls <= u.busy {
|
||||||
|
return errors.Join(ErrTryAgain, errors.New("another controller is working the plans"))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *manyTimesBusy) called() int {
|
||||||
|
u.mu.Lock()
|
||||||
|
defer u.mu.Unlock()
|
||||||
|
return u.calls
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A merge held more than five times is still handed over and recorded** (issue 973, novox/hq ADR 0407 decision
|
||||||
|
// 3): the merges have a durable consumer of their own with no max-deliver, so they never go to DEAD_LETTERS to wait
|
||||||
|
// for a person; the delay grows, and is capped.
|
||||||
|
func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) {
|
||||||
|
first, most := MergeHeldFirst, MergeHeldMost
|
||||||
|
MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond
|
||||||
|
t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most })
|
||||||
|
js := aBus(t)
|
||||||
|
up := &manyTimesBusy{busy: 8}
|
||||||
|
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
|
||||||
|
if err := s.Follows(up); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go func() { _ = s.Serve(ctx) }()
|
||||||
|
eventually(t, "the merges consumer being made", func() bool {
|
||||||
|
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
|
||||||
|
return err == nil
|
||||||
|
})
|
||||||
|
|
||||||
|
body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"})
|
||||||
|
if _, err := js.Context().Publish(MergedSubject, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 })
|
||||||
|
eventually(t, "the merge being acknowledged", func() bool {
|
||||||
|
info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
|
||||||
|
return err == nil && info.NumAckPending == 0 && info.NumPending == 0
|
||||||
|
})
|
||||||
|
if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 {
|
||||||
|
t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The delay a held merge waits grows from the first to the most and stays there.
|
||||||
|
func TestAHeldMergesDelayGrowsAndIsCapped(t *testing.T) {
|
||||||
|
var got []time.Duration
|
||||||
|
for _, held := range []time.Duration{0, 4 * time.Second, 10 * time.Second, 25 * time.Second, 70 * time.Second,
|
||||||
|
200 * time.Second, 20 * time.Minute} {
|
||||||
|
got = append(got, MergeRedeliverAfter(held))
|
||||||
|
}
|
||||||
|
want := []time.Duration{5 * time.Second, 5 * time.Second, 10 * time.Second, 25 * time.Second, 70 * time.Second,
|
||||||
|
200 * time.Second, 5 * time.Minute}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Errorf("held %d: waits %v; want %v", i, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+38
-3
@@ -636,8 +636,25 @@ func short(commit string) string {
|
|||||||
return commit
|
return commit
|
||||||
}
|
}
|
||||||
|
|
||||||
// sourceMoved acts on the forge's announcement of a merge. Taken whatever happens: a build that
|
// MergeHeldFirst is how long a merge the controller could not record yet waits before the bus hands it over
|
||||||
// fails is reported by the build itself, and re-delivering the merge would only re-fail it.
|
// again, and MergeHeldMost the longest it ever waits: the delay grows with how long the merge has been held, from
|
||||||
|
// the first to the most (novox/hq ADR 0407 decision 3). Variables so a test need not wait.
|
||||||
|
var (
|
||||||
|
MergeHeldFirst = 5 * time.Second
|
||||||
|
MergeHeldMost = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// MergeRedeliverAfter is the delay for a merge held for the time given: that time, so the delay about doubles
|
||||||
|
// with each hold, between MergeHeldFirst and MergeHeldMost.
|
||||||
|
func MergeRedeliverAfter(heldFor time.Duration) time.Duration {
|
||||||
|
return min(max(heldFor, MergeHeldFirst), MergeHeldMost)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceMoved acts on the forge's announcement of a merge. **A merge is never dropped** (issue 921, novox/hq ADR
|
||||||
|
// 0407 decision 3): one the controller could not record for now (the plans held by another controller, the store
|
||||||
|
// away: ErrTryAgain) is held and handed over again until it is recorded; the controller giving up on it would
|
||||||
|
// lose a change for good. Anything else is an answer and is taken: a build that fails is reported by the build
|
||||||
|
// itself, and re-delivering the merge would only re-fail it.
|
||||||
func (s *Server) sourceMoved(ctx context.Context, m Control) {
|
func (s *Server) sourceMoved(ctx context.Context, m Control) {
|
||||||
var moved SourceMoved
|
var moved SourceMoved
|
||||||
if err := json.Unmarshal(m.Body(), &moved); err != nil {
|
if err := json.Unmarshal(m.Body(), &moved); err != nil {
|
||||||
@@ -650,7 +667,25 @@ func (s *Server) sourceMoved(ctx context.Context, m Control) {
|
|||||||
_ = m.Took()
|
_ = m.Took()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := s.upgrader.SourceMoved(ctx, moved); err != nil {
|
err := s.upgrader.SourceMoved(ctx, moved)
|
||||||
|
// Left unsettled when the controller is ending, whatever error that gave (the plans' hold returns a bare
|
||||||
|
// context.Canceled): the bus hands it to whatever consumes next, and taking it would lose the merge.
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil && errors.Is(err, ErrTryAgain) {
|
||||||
|
// Also left unsettled when the bus will not hold it.
|
||||||
|
delay := MergeRedeliverAfter(m.HeldFor())
|
||||||
|
if held := m.Hold(delay); held != nil {
|
||||||
|
s.log.Printf("%s/%s moved to %.8s and could not be held (%v): %v; left for the bus to hand over again",
|
||||||
|
moved.Owner, moved.Repo, moved.Commit, held, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.log.Printf("%s/%s moved to %.8s and is held: %v; handed over again in %s", moved.Owner, moved.Repo,
|
||||||
|
moved.Commit, err, delay)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
s.log.Printf("%s/%s moved to %.8s and the mesh could not act on it: %v", moved.Owner, moved.Repo, moved.Commit, err)
|
s.log.Printf("%s/%s moved to %.8s and the mesh could not act on it: %v", moved.Owner, moved.Repo, moved.Commit, err)
|
||||||
}
|
}
|
||||||
_ = m.Took()
|
_ = m.Took()
|
||||||
|
|||||||
@@ -106,3 +106,61 @@ func TestAMessageHandledDuringShutdownIsLeftForTheBus(t *testing.T) {
|
|||||||
t.Fatalf("a build result handled during shutdown was settled, and so lost: %+v", *to)
|
t.Fatalf("a build result handled during shutdown was settled, and so lost: %+v", *to)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A merge the controller could not record for now (the plans held by another controller, the store away) is held
|
||||||
|
// and handed over again, never taken: the bus keeps it until it is recorded (issue 921). Any other failure is an
|
||||||
|
// answer and is taken, as before.
|
||||||
|
func TestAMergeThatCouldNotBeRecordedNowIsHeldNotDropped(t *testing.T) {
|
||||||
|
merged := SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
err error
|
||||||
|
want func(*settled) bool
|
||||||
|
}{
|
||||||
|
{"the plans held by another controller", errors.Join(ErrTryAgain, errors.New("another controller is working the plans")),
|
||||||
|
func(s *settled) bool { return s.unsettled() }},
|
||||||
|
{"the store away", errors.Join(ErrTryAgain, restarting), func(s *settled) bool { return s.unsettled() }},
|
||||||
|
{"cannot act", errors.New("no such module"), func(s *settled) bool { return s.acked && !s.nacked }},
|
||||||
|
{"kept", nil, func(s *settled) bool { return s.acked && !s.nacked }},
|
||||||
|
} {
|
||||||
|
s, in := serving()
|
||||||
|
s.upgrader = upgradesWith{err: c.err}
|
||||||
|
to := &settled{}
|
||||||
|
held := in.sends(t, to, KindSourceMoved, merged).(*fakeControl)
|
||||||
|
s.act(context.Background(), held)
|
||||||
|
if !c.want(to) {
|
||||||
|
t.Errorf("%s: a merge was settled as %+v", c.what, *to)
|
||||||
|
}
|
||||||
|
if c.err != nil && errors.Is(c.err, ErrTryAgain) && len(in.held) != 1 {
|
||||||
|
t.Errorf("%s: the merge was not held to be handed over again", c.what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge being held when the controller is replaced (its context ended) or when the bus will not hold it is left
|
||||||
|
// unsettled, never taken: the bus hands it to whatever consumes next (review of issue 921).
|
||||||
|
func TestAMergeIsLeftUnsettledWhenTheControllerEndsOrTheHoldFails(t *testing.T) {
|
||||||
|
merged := SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}
|
||||||
|
try := errors.Join(ErrTryAgain, errors.New("another controller is working the plans"))
|
||||||
|
|
||||||
|
// The plans' hold returns a bare context.Canceled when the controller is replaced, not ErrTryAgain.
|
||||||
|
s, in := serving()
|
||||||
|
s.upgrader = upgradesWith{err: context.Canceled}
|
||||||
|
to := &settled{}
|
||||||
|
ended, end := context.WithCancel(context.Background())
|
||||||
|
end()
|
||||||
|
s.act(ended, in.sends(t, to, KindSourceMoved, merged))
|
||||||
|
if !to.unsettled() {
|
||||||
|
t.Errorf("a merge was settled as %+v when the controller's context ended", *to)
|
||||||
|
}
|
||||||
|
|
||||||
|
s, in = serving()
|
||||||
|
s.upgrader = upgradesWith{err: try}
|
||||||
|
to = &settled{}
|
||||||
|
m := in.sends(t, to, KindSourceMoved, merged).(*fakeControl)
|
||||||
|
m.holdFails = errors.New("the bus will not hold it")
|
||||||
|
s.act(context.Background(), m)
|
||||||
|
if !to.unsettled() {
|
||||||
|
t.Errorf("a merge was settled as %+v when it could not be held", *to)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package link
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"reflect"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -30,25 +29,8 @@ func TestATrysStartNamesNoCommitTheForgeCouldMatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A try stays behind every delivery and every pull request's check, however long it has waited: the ten
|
// An id that is a pattern never passes for a build's id.
|
||||||
// minutes that put a check ahead of later deliveries never promote a try (novox/hq issue 686).
|
func TestAnIdThatIsAPatternIsNoBuildId(t *testing.T) {
|
||||||
func TestATryIsNeverHandedOutBeforeADeliveryOrAPullRequestsCheck(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
|
||||||
try := func(seq uint64, id string, asked time.Time) QueuedAsk {
|
|
||||||
a := anAsk(seq, id, asked, true)
|
|
||||||
a.Request.Check.Try = true
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
waiting := []QueuedAsk{
|
|
||||||
try(1, "try-old", now.Add(-40*time.Minute)),
|
|
||||||
try(2, "try-young", now.Add(-time.Minute)),
|
|
||||||
anAsk(3, "check-young", now.Add(-2*time.Minute), true),
|
|
||||||
anAsk(4, "delivery", now.Add(-time.Minute), false),
|
|
||||||
}
|
|
||||||
want := []string{"delivery", "check-young", "try-old", "try-young"}
|
|
||||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("handed out as %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") {
|
if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") {
|
||||||
t.Fatal("an id that is a pattern passes for a build's id")
|
t.Fatal("an id that is a pattern passes for a build's id")
|
||||||
}
|
}
|
||||||
|
|||||||
Vendored
+11
@@ -8,6 +8,7 @@
|
|||||||
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
|
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
|
||||||
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
|
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
|
||||||
./cmd/mesh-controller TestAPlanFollowedFromTheStoreAnswersWhenItMoves
|
./cmd/mesh-controller TestAPlanFollowedFromTheStoreAnswersWhenItMoves
|
||||||
|
./cmd/mesh-controller TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself
|
||||||
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
|
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
|
||||||
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
|
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
|
||||||
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
|
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
|
||||||
@@ -126,3 +127,13 @@
|
|||||||
./cmd/mesh-controller TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn
|
./cmd/mesh-controller TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn
|
||||||
./cmd/mesh-controller TestAGroupsCompositionIntoAReleaseBranchIsJudgedByItsOwnController
|
./cmd/mesh-controller TestAGroupsCompositionIntoAReleaseBranchIsJudgedByItsOwnController
|
||||||
./cmd/mesh-controller TestAGroupsCompositionJudgedWithTheValidatorIntoAReleaseBranchIsJudgedByItsOwnController
|
./cmd/mesh-controller TestAGroupsCompositionJudgedWithTheValidatorIntoAReleaseBranchIsJudgedByItsOwnController
|
||||||
|
./cmd/mesh-controller TestAssigningTheClientIssuesItsAccount
|
||||||
|
./cmd/mesh-controller TestAnUnrelatedBatchWalksBesideAnOpenWalk
|
||||||
|
./cmd/mesh-controller TestARelatedBatchWaitsBehindTheOpenWalk
|
||||||
|
./cmd/mesh-controller TestAnOwnPathBatchNeverWalksBeside
|
||||||
|
./cmd/mesh-controller TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried
|
||||||
|
./cmd/mesh-controller TestABatchWindowClosesByTheExpiryOfItsKey
|
||||||
|
./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid
|
||||||
|
./cmd/mesh-controller TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt
|
||||||
|
./cmd/mesh-controller TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent
|
||||||
|
./cmd/mesh-controller TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck
|
||||||
|
|||||||
Reference in New Issue
Block a user